diff --git "a/wandb/run-20240122_192233-r12o64xn/logs/debug-internal.log" "b/wandb/run-20240122_192233-r12o64xn/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240122_192233-r12o64xn/logs/debug-internal.log" @@ -0,0 +1,9157 @@ +2024-01-22 19:22:33,305 INFO StreamThr :123548 [internal.py:wandb_internal():86] W&B internal server running at pid: 123548, started at: 2024-01-22 19:22:33.305207 +2024-01-22 19:22:33,307 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status +2024-01-22 19:22:33,309 INFO WriterThread:123548 [datastore.py:open_for_write():85] open: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/run-r12o64xn.wandb +2024-01-22 19:22:33,310 DEBUG SenderThread:123548 [sender.py:send():382] send: header +2024-01-22 19:22:33,461 DEBUG SenderThread:123548 [sender.py:send():382] send: run +2024-01-22 19:22:33,771 INFO SenderThread:123548 [dir_watcher.py:__init__():211] watching files in: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files +2024-01-22 19:22:33,772 INFO SenderThread:123548 [sender.py:_start_run_threads():1136] run started: r12o64xn with start time 1705947753.304818 +2024-01-22 19:22:33,780 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: check_version +2024-01-22 19:22:33,781 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: check_version +2024-01-22 19:22:33,813 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: run_start +2024-01-22 19:22:33,826 DEBUG HandlerThread:123548 [system_info.py:__init__():32] System info init +2024-01-22 19:22:33,827 DEBUG HandlerThread:123548 [system_info.py:__init__():47] System info init done +2024-01-22 19:22:33,827 INFO HandlerThread:123548 [system_monitor.py:start():194] Starting system monitor +2024-01-22 19:22:33,827 INFO SystemMonitor:123548 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-01-22 19:22:33,828 INFO HandlerThread:123548 [system_monitor.py:probe():214] Collecting system info +2024-01-22 19:22:33,828 INFO SystemMonitor:123548 [interfaces.py:start():190] Started cpu monitoring +2024-01-22 19:22:33,829 INFO SystemMonitor:123548 [interfaces.py:start():190] Started disk monitoring +2024-01-22 19:22:33,829 INFO SystemMonitor:123548 [interfaces.py:start():190] Started gpu monitoring +2024-01-22 19:22:33,830 INFO SystemMonitor:123548 [interfaces.py:start():190] Started memory monitoring +2024-01-22 19:22:33,830 INFO SystemMonitor:123548 [interfaces.py:start():190] Started network monitoring +2024-01-22 19:22:33,871 DEBUG HandlerThread:123548 [system_info.py:probe():196] Probing system +2024-01-22 19:22:33,876 DEBUG HandlerThread:123548 [system_info.py:_probe_git():181] Probing git +2024-01-22 19:22:33,887 DEBUG HandlerThread:123548 [system_info.py:_probe_git():189] Probing git done +2024-01-22 19:22:33,887 DEBUG HandlerThread:123548 [system_info.py:probe():244] Probing system done +2024-01-22 19:22:33,887 DEBUG HandlerThread:123548 [system_monitor.py:probe():223] {'os': 'Linux-5.4.0-166-generic-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2024-01-22T18:22:33.871121', 'startedAt': '2024-01-22T18:22:33.300573', 'docker': None, 'cuda': None, 'args': ('--model_name_or_path=openai/whisper-large-v3', '--dataset_name=mozilla-foundation/common_voice_16_0', '--dataset_config_name=mn', '--language=mongolian', '--train_split_name=train+validation', '--eval_split_name=test', '--num_train_epochs=10', '--output_dir=./', '--per_device_train_batch_size=32', '--per_device_eval_batch_size=32', '--logging_steps=25', '--learning_rate=1e-4', '--warmup_steps=500', '--evaluation_strategy=steps', '--eval_steps=500', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--streaming=False', '--gradient_checkpointing', '--fp16', '--fp16_full_eval', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--use_auth_token', '--push_to_hub'), 'state': 'running', 'program': 'run_speech_recognition_seq2seq.py', 'codePathLocal': 'run_speech_recognition_seq2seq.py', 'codePath': 'run_speech_recognition_seq2seq.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/whisper-large-v3-ft-cv16-mn', 'commit': '10fd9dc687df54a952f755348a2a48d40389d8b3'}, 'email': 'sanchit@huggingface.co', 'root': '/home/sanchit/whisper-large-v3-ft-cv16-mn', 'host': 'hf-dgx-01', 'username': 'sanchit', 'executable': '/home/sanchit/hf/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3214.9774531250005, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 3230.366, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.183, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.187, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.236, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.43, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.489, 'min': 1500.0, 'max': 2250.0}, {'current': 3266.476, 'min': 1500.0, 'max': 2250.0}, {'current': 3210.048, 'min': 1500.0, 'max': 2250.0}, {'current': 3257.545, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.303, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.427, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.426, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.587, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.399, 'min': 1500.0, 'max': 2250.0}, {'current': 2904.142, 'min': 1500.0, 'max': 2250.0}, {'current': 3211.356, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.632, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.265, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.364, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.958, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.642, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.096, 'min': 1500.0, 'max': 2250.0}, {'current': 2937.066, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.016, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.477, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.63, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.084, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.593, 'min': 1500.0, 'max': 2250.0}, {'current': 3254.989, 'min': 1500.0, 'max': 2250.0}, {'current': 3262.86, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.642, 'min': 1500.0, 'max': 2250.0}, {'current': 3265.556, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.61, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.422, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.412, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.211, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.87, 'min': 1500.0, 'max': 2250.0}, {'current': 3253.496, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.377, 'min': 1500.0, 'max': 2250.0}, {'current': 3258.916, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.239, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.375, 'min': 1500.0, 'max': 2250.0}, {'current': 1905.767, 'min': 1500.0, 'max': 2250.0}, {'current': 3258.181, 'min': 1500.0, 'max': 2250.0}, {'current': 3257.87, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.345, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.017, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.398, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.362, 'min': 1500.0, 'max': 2250.0}, {'current': 3261.943, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.11, 'min': 1500.0, 'max': 2250.0}, {'current': 3268.252, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.777, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.768, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.488, 'min': 1500.0, 'max': 2250.0}, {'current': 3259.674, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.471, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.291, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.291, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.294, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.37, 'min': 1500.0, 'max': 2250.0}, {'current': 3242.031, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.502, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.338, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.627, 'min': 1500.0, 'max': 2250.0}, {'current': 3240.767, 'min': 1500.0, 'max': 2250.0}, {'current': 3258.625, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.795, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.372, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.617, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.182, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.277, 'min': 1500.0, 'max': 2250.0}, {'current': 3242.024, 'min': 1500.0, 'max': 2250.0}, {'current': 3254.417, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.75, 'min': 1500.0, 'max': 2250.0}, {'current': 3257.086, 'min': 1500.0, 'max': 2250.0}, {'current': 3267.203, 'min': 1500.0, 'max': 2250.0}, {'current': 3268.072, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.431, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.159, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.012, 'min': 1500.0, 'max': 2250.0}, {'current': 3252.771, 'min': 1500.0, 'max': 2250.0}, {'current': 3260.228, 'min': 1500.0, 'max': 2250.0}, {'current': 3256.431, 'min': 1500.0, 'max': 2250.0}, {'current': 3257.347, 'min': 1500.0, 'max': 2250.0}, {'current': 3243.546, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.212, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.644, 'min': 1500.0, 'max': 2250.0}, {'current': 3251.766, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.918, 'min': 1500.0, 'max': 2250.0}, {'current': 3115.391, 'min': 1500.0, 'max': 2250.0}, {'current': 3259.897, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.289, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.461, 'min': 1500.0, 'max': 2250.0}, {'current': 3256.351, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.418, 'min': 1500.0, 'max': 2250.0}, {'current': 3264.353, 'min': 1500.0, 'max': 2250.0}, {'current': 3242.239, 'min': 1500.0, 'max': 2250.0}, {'current': 3244.06, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.87, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.265, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.34, 'min': 1500.0, 'max': 2250.0}, {'current': 3249.998, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.476, 'min': 1500.0, 'max': 2250.0}, {'current': 3267.122, 'min': 1500.0, 'max': 2250.0}, {'current': 3266.894, 'min': 1500.0, 'max': 2250.0}, {'current': 1654.307, 'min': 1500.0, 'max': 2250.0}, {'current': 3245.553, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.145, 'min': 1500.0, 'max': 2250.0}, {'current': 3238.04, 'min': 1500.0, 'max': 2250.0}, {'current': 3260.439, 'min': 1500.0, 'max': 2250.0}, {'current': 3246.639, 'min': 1500.0, 'max': 2250.0}, {'current': 3242.546, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.1, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.153, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.314, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.139, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.295, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.63, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.515, 'min': 1500.0, 'max': 2250.0}, {'current': 3247.69, 'min': 1500.0, 'max': 2250.0}, {'current': 3256.758, 'min': 1500.0, 'max': 2250.0}, {'current': 3250.459, 'min': 1500.0, 'max': 2250.0}, {'current': 3263.811, 'min': 1500.0, 'max': 2250.0}, {'current': 3253.981, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.603, 'min': 1500.0, 'max': 2250.0}, {'current': 3248.634, 'min': 1500.0, 'max': 2250.0}, {'current': 3237.134, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 1757.8785285949707, 'used': 1587.8179626464844}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 5, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA DGX Display', 'memory_total': 4294967296}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5397186279297}} +2024-01-22 19:22:33,888 INFO HandlerThread:123548 [system_monitor.py:probe():224] Finished collecting system info +2024-01-22 19:22:33,888 INFO HandlerThread:123548 [system_monitor.py:probe():227] Publishing system info +2024-01-22 19:22:33,888 DEBUG HandlerThread:123548 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2024-01-22 19:22:33,888 DEBUG HandlerThread:123548 [system_info.py:_save_pip():68] Saving pip packages done +2024-01-22 19:22:33,890 INFO HandlerThread:123548 [system_monitor.py:probe():229] Finished publishing system info +2024-01-22 19:22:33,894 DEBUG SenderThread:123548 [sender.py:send():382] send: files +2024-01-22 19:22:33,894 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2024-01-22 19:22:33,898 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:33,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:22:33,899 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:22:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:22:33,912 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:34,111 DEBUG SenderThread:123548 [sender.py:send():382] send: telemetry +2024-01-22 19:22:34,111 DEBUG SenderThread:123548 [sender.py:send():382] send: config +2024-01-22 19:22:34,112 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 19:22:34,113 DEBUG SenderThread:123548 [sender.py:send():382] send: telemetry +2024-01-22 19:22:34,113 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 19:22:34,113 WARNING SenderThread:123548 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2024-01-22 19:22:34,113 DEBUG SenderThread:123548 [sender.py:send():382] send: telemetry +2024-01-22 19:22:34,367 INFO wandb-upload_0:123548 [upload_job.py:push():131] Uploaded file /tmp/tmp31zftf2iwandb/su843a10-wandb-metadata.json +2024-01-22 19:22:34,773 INFO Thread-12 :123548 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-metadata.json +2024-01-22 19:22:34,773 INFO Thread-12 :123548 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/requirements.txt +2024-01-22 19:22:34,774 INFO Thread-12 :123548 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:35,942 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:35,959 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:36,773 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:37,991 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:38,004 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:38,547 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:22:40,028 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:40,044 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:40,774 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:42,068 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:42,085 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:43,580 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:22:44,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:44,145 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:44,775 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:46,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:46,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:48,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:48,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:48,777 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:48,898 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:22:48,899 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:22:48,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:22:49,061 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:22:50,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:50,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:52,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:52,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:52,778 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:54,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:54,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:54,485 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:22:56,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:56,386 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:56,779 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:22:58,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:22:58,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:00,434 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:00,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:00,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:00,780 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:02,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:02,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:03,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:23:03,899 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:23:03,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:23:04,781 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:04,937 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:23:04,961 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:04,974 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:06,037 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:06,782 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/config.yaml +2024-01-22 19:23:06,999 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:07,015 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:08,783 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:09,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:09,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:11,090 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:11,105 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:11,285 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:12,784 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:13,136 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:13,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:15,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:15,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:16,785 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:17,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:17,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:17,231 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:18,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:23:18,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:23:18,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:23:19,262 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:19,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:20,786 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:21,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:21,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:23,189 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:23,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:23,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:24,787 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:25,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:25,390 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:26,788 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:27,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:27,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:29,178 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:29,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:29,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:30,789 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:31,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:31,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:33,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:33,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:33,830 DEBUG SystemMonitor:123548 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-01-22 19:23:33,833 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:23:33,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:23:33,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:23:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:23:34,790 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:35,099 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:35,944 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:23:35,967 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:35,978 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:38,003 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:38,016 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:38,792 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:40,042 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:40,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:41,075 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:42,085 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:42,102 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:42,793 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:44,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:44,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:46,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:46,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:46,794 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:47,052 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:48,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:48,221 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:48,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:23:48,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:23:48,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:23:50,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:50,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:50,796 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:52,280 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:52,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:52,998 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:23:54,317 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:54,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:54,797 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:56,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:56,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:58,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:58,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:23:58,798 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:23:58,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:00,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:00,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:02,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:02,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:02,799 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:03,835 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:24:03,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:24:03,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:24:03,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:24:04,044 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:04,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:04,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:06,800 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:06,918 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:24:06,941 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:06,961 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:08,985 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:09,004 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:09,832 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:10,801 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:11,026 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:11,046 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:13,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:13,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:13,875 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:24:13,877 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 19:24:13,877 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 19:24:13,877 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 19:24:13,877 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:24:13,878 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:24:13,878 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:24:14,803 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:14,803 INFO Thread-12 :123548 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:24:14,884 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:15,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:15,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:15,803 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/config.yaml +2024-01-22 19:24:16,803 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:17,176 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:17,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:18,804 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:18,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:24:18,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:24:18,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:24:19,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:19,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:20,128 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:21,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:21,267 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:22,805 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:23,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:23,303 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:25,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:25,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:25,803 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:26,807 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:27,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:27,380 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:29,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:29,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:30,806 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:30,808 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:31,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:31,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:33,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:33,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:33,836 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:24:33,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:24:33,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:24:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:24:34,809 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:35,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:35,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:36,022 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:37,928 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:24:37,958 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:37,971 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:38,810 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:39,996 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:40,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:41,736 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:42,034 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:42,048 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:42,811 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:44,077 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:44,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:46,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:46,142 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:46,813 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:47,701 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:48,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:48,186 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:48,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:24:48,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:24:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:24:50,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:50,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:50,814 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:52,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:52,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:53,682 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:24:54,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:54,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:54,815 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:56,335 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:56,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:58,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:58,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:24:58,817 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:24:59,675 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:00,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:00,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:02,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:02,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:02,818 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:03,839 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:25:03,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:25:03,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:25:03,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:25:04,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:04,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:05,059 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:06,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:06,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:06,819 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:08,942 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:25:08,967 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:08,983 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:10,588 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:10,821 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:11,010 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:11,026 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:13,065 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:13,079 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:14,822 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:15,102 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:15,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:15,592 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:17,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:17,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:18,823 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:18,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:25:18,900 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:25:18,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:25:19,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:19,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:21,042 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:21,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:21,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:22,824 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:23,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:23,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:25,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:25,335 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:26,604 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:26,826 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:27,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:27,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:29,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:29,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:30,827 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:31,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:31,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:31,611 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:33,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:33,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:33,840 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:25:33,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:25:33,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:25:33,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:25:34,828 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:35,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:35,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:37,126 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:37,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:37,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:38,830 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:39,965 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:25:40,006 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:40,023 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:42,047 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:42,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:42,590 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:42,831 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:44,086 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:44,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:46,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:46,144 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:46,833 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:48,181 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:48,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:48,585 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:48,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:25:48,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:25:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:25:50,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:50,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:50,834 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:52,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:52,275 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:53,631 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:25:53,633 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:25:53,633 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:25:53,633 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:25:53,635 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:25:53,835 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:25:54,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:54,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:54,835 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:56,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:56,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:56,836 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:58,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:58,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:25:58,837 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:25:59,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:00,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:00,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:02,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:02,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:02,838 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:03,842 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:26:03,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:26:03,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:26:03,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:26:04,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:04,503 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:05,072 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:06,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:06,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:06,839 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:08,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:08,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:10,574 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:10,841 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:10,975 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:26:11,003 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:11,014 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:13,040 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:13,052 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:14,842 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:15,076 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:15,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:16,560 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:17,112 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:17,131 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:18,843 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:18,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:26:18,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:26:18,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:26:19,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:19,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:21,204 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:21,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:22,551 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:22,845 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:23,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:23,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:25,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:25,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:26,846 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:27,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:27,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:28,538 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:29,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:29,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:30,847 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:31,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:31,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:33,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:33,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:33,843 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:26:33,845 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:33,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:26:33,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:26:33,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:26:34,849 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:35,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:35,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:37,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:37,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:38,850 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:39,507 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:39,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:39,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:41,999 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:26:42,026 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:42,040 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:42,851 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:44,068 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:44,079 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:45,478 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:46,114 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:46,126 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:46,853 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:48,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:48,169 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:48,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:26:48,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:26:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:26:50,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:50,213 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:50,854 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:51,469 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:52,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:52,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:54,287 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:54,306 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:54,855 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:26:56,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:56,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:57,455 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:26:58,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:58,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:26:58,856 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:00,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:00,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:02,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:02,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:02,482 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:02,857 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:03,845 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:27:03,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:27:03,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:27:03,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:27:04,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:04,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:06,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:06,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:06,859 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:08,479 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:08,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:08,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:10,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:10,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:10,860 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:13,022 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:27:13,047 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:13,060 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:14,463 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:14,861 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:15,082 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:15,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:17,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:17,145 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:18,863 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:18,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:27:18,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:27:18,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:27:19,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:19,187 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:20,119 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:21,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:21,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:22,864 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:23,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:23,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:25,303 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:25,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:25,426 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:26,865 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:27,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:27,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:29,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:29,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:30,430 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:30,866 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:31,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:31,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:33,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:33,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:33,504 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:27:33,505 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:27:33,506 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:27:33,506 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:27:33,847 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:27:33,867 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:27:33,900 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:27:33,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:27:33,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:27:34,868 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:35,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:35,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:36,059 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:37,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:37,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:38,869 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:39,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:39,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:41,454 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:41,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:41,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:42,870 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:44,042 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:27:44,069 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:44,086 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:46,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:46,127 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:46,502 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:46,871 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:48,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:48,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:48,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:27:48,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:27:48,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:27:50,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:50,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:50,873 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:51,528 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:52,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:52,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:54,269 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:54,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:54,874 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:27:56,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:56,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:57,510 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:27:58,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:58,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:27:58,875 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:00,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:00,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:02,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:02,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:02,512 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:02,876 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:03,848 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:28:03,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:28:03,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:28:03,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:28:04,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:04,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:06,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:06,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:06,878 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:07,534 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:08,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:08,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:10,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:10,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:10,879 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:12,537 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:12,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:12,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:14,880 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:15,059 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:28:15,088 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:15,099 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:17,122 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:17,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:18,528 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:18,882 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:18,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:28:18,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:28:18,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:28:19,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:19,181 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:21,207 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:21,225 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:22,883 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:23,251 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:23,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:23,559 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:25,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:25,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:26,884 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:27,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:27,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:29,380 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:29,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:29,559 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:30,885 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:31,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:31,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:33,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:33,473 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:33,850 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:28:33,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:28:33,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:28:33,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:28:34,887 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:35,041 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:35,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:35,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:37,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:37,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:38,888 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:39,582 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:39,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:40,615 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:41,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:41,630 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:42,889 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:43,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:43,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:45,638 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:46,108 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:28:46,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:46,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:46,890 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:48,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:48,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:48,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:28:48,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:28:48,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:28:50,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:50,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:50,892 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:51,630 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:52,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:52,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:54,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:54,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:54,893 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:28:56,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:56,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:56,677 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:28:58,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:58,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:28:58,894 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:00,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:00,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:01,683 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:02,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:02,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:02,895 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:03,851 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:29:03,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:29:03,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:29:03,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:29:04,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:04,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:06,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:06,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:06,708 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:06,897 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:08,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:08,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:10,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:10,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:10,898 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:11,733 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:12,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:12,700 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:13,806 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:29:13,807 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:29:13,808 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:29:13,808 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:29:13,898 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:29:14,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:14,742 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:14,899 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:16,809 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:16,899 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:17,132 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:29:17,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:17,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:18,900 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:18,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:29:18,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:29:18,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:29:19,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:19,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:21,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:21,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:22,781 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:22,901 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:23,285 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:23,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:25,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:25,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:26,903 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:27,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:27,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:28,779 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:29,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:29,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:30,904 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:31,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:31,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:33,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:33,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:33,808 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:33,853 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:29:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:29:33,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:29:33,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:29:34,906 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:35,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:35,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:37,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:37,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:38,830 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:38,907 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:39,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:39,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:41,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:41,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:42,908 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:43,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:43,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:43,830 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:45,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:45,753 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:46,909 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:48,141 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:29:48,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:48,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:48,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:29:48,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:29:48,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:29:49,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:50,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:50,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:50,910 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:52,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:52,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:54,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:54,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:54,862 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:29:54,911 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:29:56,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:56,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:58,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:58,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:29:58,912 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:00,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:00,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:00,859 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:02,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:02,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:02,913 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:03,854 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:30:03,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:30:03,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:30:03,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:30:04,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:04,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:05,860 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:06,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:06,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:06,914 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:08,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:08,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:10,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:10,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:10,915 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:11,636 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:12,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:12,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:12,916 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:14,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:14,717 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:16,663 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:16,745 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:16,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:16,917 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:18,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:30:18,902 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:30:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:30:19,155 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:30:19,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:19,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:20,919 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:21,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:21,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:21,669 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:23,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:23,280 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:24,920 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:25,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:25,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:27,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:27,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:27,669 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:28,921 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:29,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:29,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:31,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:31,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:32,717 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:32,922 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:33,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:33,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:33,856 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:30:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:30:33,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:30:33,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:30:35,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:35,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:36,924 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:37,572 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:37,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:38,715 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:39,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:39,627 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:40,925 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:41,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:41,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:43,716 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:43,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:43,741 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:44,926 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:45,802 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:45,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:47,846 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:47,860 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:48,739 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:48,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:30:48,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:30:48,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:30:48,927 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:50,245 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:30:50,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:50,287 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:51,789 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:30:51,791 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:30:51,791 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:30:51,791 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:30:51,929 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:30:52,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:52,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:52,929 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:53,792 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:30:54,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:54,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:56,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:56,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:56,930 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:30:58,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:58,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:30:59,751 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:00,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:00,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:00,932 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:02,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:02,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:03,857 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:31:03,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:31:03,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:31:03,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:31:04,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:04,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:04,933 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:05,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:06,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:06,600 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:08,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:08,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:08,934 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:10,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:10,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:10,760 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:12,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:12,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:12,935 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:14,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:14,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:15,764 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:16,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:16,812 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:16,936 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:18,836 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:18,854 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:18,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:31:18,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:31:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:31:20,797 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:20,938 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:21,253 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:31:21,278 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:21,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:23,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:23,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:24,939 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:25,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:25,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:25,823 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:27,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:27,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:28,940 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:29,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:29,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:30,830 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:31,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:31,495 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:32,941 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:33,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:33,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:33,859 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:31:33,902 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:31:33,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:31:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:31:35,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:35,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:36,815 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:36,943 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:37,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:37,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:39,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:39,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:40,944 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:41,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:41,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:41,854 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:43,713 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:43,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:44,945 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:45,758 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:45,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:46,887 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:47,800 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:47,813 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:31:48,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:31:48,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:31:48,946 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:49,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:49,854 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:51,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:52,259 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:31:52,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:52,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:52,948 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:54,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:54,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:56,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:56,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:56,949 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:31:57,178 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:31:58,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:31:58,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:00,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:00,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:00,950 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:02,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:02,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:03,157 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:03,861 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:32:03,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:32:03,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:32:03,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:32:04,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:04,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:04,951 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:06,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:06,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:08,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:08,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:09,157 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:09,953 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:10,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:10,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:12,686 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:12,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:13,954 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:14,168 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:14,728 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:14,746 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:16,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:16,787 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:17,955 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:18,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:18,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:18,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:32:18,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:32:18,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:32:20,078 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:20,851 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:20,863 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:21,956 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:23,270 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:32:23,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:23,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:25,169 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:25,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:25,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:25,957 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:27,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:27,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:29,420 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:29,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:29,959 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:30,208 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:31,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:31,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:32,289 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:32:32,290 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:32:32,290 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:32:32,291 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:32:32,960 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:32,960 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:32:33,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:33,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:33,862 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:32:33,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:32:33,903 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:32:33,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:32:33,960 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:35,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:35,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:36,116 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:37,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:37,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:37,961 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:39,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:39,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:41,319 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:41,668 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:41,681 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:41,962 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:43,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:43,720 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:45,747 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:45,758 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:45,963 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:47,316 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:47,784 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:47,801 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:32:48,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:32:48,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:32:49,826 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:49,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:49,965 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:51,865 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:51,878 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:52,362 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:53,966 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:54,271 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:32:54,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:54,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:56,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:56,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:57,364 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:32:57,968 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:32:58,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:32:58,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:00,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:00,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:01,969 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:02,368 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:02,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:02,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:03,863 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:33:03,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:33:03,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:33:03,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:33:04,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:04,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:05,970 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:06,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:06,572 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:07,381 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:08,595 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:08,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:09,971 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:10,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:10,649 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:12,417 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:12,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:12,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:13,972 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:14,716 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:14,727 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:16,755 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:16,766 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:17,464 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:17,974 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:18,793 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:18,805 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:18,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:33:18,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:33:18,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:33:20,834 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:20,846 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:21,975 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:22,517 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:22,873 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:22,884 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:25,280 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:33:25,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:25,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:25,976 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:27,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:27,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:27,547 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:29,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:29,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:29,978 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:31,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:31,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:32,557 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:33,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:33,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:33,865 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:33:33,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:33:33,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:33:33,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:33:33,979 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:35,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:35,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:37,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:37,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:37,578 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:37,980 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:39,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:39,608 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:41,635 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:41,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:41,981 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:42,615 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:43,668 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:43,681 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:45,708 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:45,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:45,983 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:47,628 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:47,748 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:47,759 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:48,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:33:48,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:33:48,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:33:49,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:49,806 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:49,984 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:51,837 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:51,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:52,672 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:53,880 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:53,892 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:53,985 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:56,281 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:33:56,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:56,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:57,687 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:33:57,986 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:33:58,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:33:58,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:00,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:00,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:01,988 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:02,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:02,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:02,692 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:03,866 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:34:03,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:34:03,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:34:03,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:34:04,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:04,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:05,989 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:06,521 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:06,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:07,734 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:08,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:08,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:09,990 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:10,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:10,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:12,633 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:12,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:12,772 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:12,856 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:34:12,857 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:34:12,858 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:34:12,858 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:34:12,991 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:34:13,991 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:14,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:14,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:14,992 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:16,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:16,735 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:17,819 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:17,993 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:18,763 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:18,775 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:18,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:34:18,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:34:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:34:20,800 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:20,816 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:21,994 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:22,839 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:22,855 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:22,868 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:24,880 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:24,892 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:25,995 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:27,290 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:34:27,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:27,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:27,873 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:29,361 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:29,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:29,996 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:31,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:31,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:32,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:33,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:33,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:33,869 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:34:33,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:34:33,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:34:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:34:33,998 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:35,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:35,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:37,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:37,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:37,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:37,999 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:39,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:39,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:41,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:41,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:42,000 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:43,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:43,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:43,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:45,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:45,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:46,001 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:47,729 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:47,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:48,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:34:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:34:48,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:34:49,095 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:49,763 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:49,774 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:51,804 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:51,816 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:52,003 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:53,843 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:53,855 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:54,004 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:54,981 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:34:55,883 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:55,905 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:58,005 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:34:58,302 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:34:58,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:58,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:34:59,986 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:00,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:00,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:02,006 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:02,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:02,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:03,870 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:35:03,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:35:03,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:35:03,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:35:04,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:04,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:05,076 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:06,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:06,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:08,008 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:08,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:08,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:10,099 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:10,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:10,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:12,009 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:12,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:12,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:14,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:14,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:15,102 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:16,010 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:16,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:16,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:18,736 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:18,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:18,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:35:18,904 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:35:18,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:35:20,011 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:20,775 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:20,786 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:21,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:22,813 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:22,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:24,013 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:24,853 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:24,865 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:26,151 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:26,889 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:26,910 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:28,014 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:29,324 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:35:29,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:29,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:31,165 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:31,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:31,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:32,017 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:33,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:33,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:33,871 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:35:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:35:33,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:35:33,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:35:35,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:35,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:36,018 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:37,132 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:37,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:37,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:39,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:39,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:40,019 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:41,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:41,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:42,226 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:43,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:43,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:44,021 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:45,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:45,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:47,243 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:47,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:47,723 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:48,022 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:48,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:35:48,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:35:48,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:35:49,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:49,760 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:51,782 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:51,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:52,023 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:52,261 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:53,356 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:35:53,358 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:35:53,358 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:35:53,358 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:35:53,828 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:53,839 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:54,023 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:35:55,023 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:55,864 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:55,878 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:56,024 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:35:57,352 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:35:57,904 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:35:57,915 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:00,025 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:00,318 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:36:00,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:00,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:02,368 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:02,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:02,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:03,873 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:36:03,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:36:03,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:36:03,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:36:04,026 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:04,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:04,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:06,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:06,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:07,403 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:08,027 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:08,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:08,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:10,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:10,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:12,028 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:12,417 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:12,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:12,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:14,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:14,622 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:16,030 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:16,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:16,662 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:17,434 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:18,684 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:18,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:36:18,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:36:18,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:36:20,031 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:20,725 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:20,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:22,490 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:22,763 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:22,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:24,032 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:24,805 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:24,818 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:26,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:26,851 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:27,515 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:28,033 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:28,876 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:28,887 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:31,287 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:36:31,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:31,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:32,034 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:32,583 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:33,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:33,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:33,875 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:36:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:36:33,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:36:33,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:36:35,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:35,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:36,035 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:37,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:37,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:37,658 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:39,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:39,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:40,037 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:41,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:41,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:42,666 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:43,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:43,595 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:44,038 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:45,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:45,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:47,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:47,689 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:47,716 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:48,039 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:48,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:36:48,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:36:48,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:36:49,714 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:49,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:51,755 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:51,774 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:52,040 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:52,761 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:53,804 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:53,824 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:55,849 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:55,861 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:56,041 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:36:57,806 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:36:57,891 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:57,902 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:59,931 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:36:59,942 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:00,043 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:02,357 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:37:02,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:02,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:02,837 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:03,877 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:37:03,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:37:03,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:37:03,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:37:04,044 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:04,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:04,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:06,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:06,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:07,889 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:08,045 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:08,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:08,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:10,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:10,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:12,046 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:12,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:12,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:12,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:14,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:14,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:16,048 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:16,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:16,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:17,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:18,698 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:18,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:37:18,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:37:18,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:37:20,049 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:20,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:20,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:22,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:22,789 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:23,018 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:24,050 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:24,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:24,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:26,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:26,862 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:28,051 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:28,087 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:28,890 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:28,907 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:30,928 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:30,942 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:32,052 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:33,112 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:33,346 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:37:33,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:33,390 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:33,878 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:37:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:37:33,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:37:33,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:37:34,205 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:37:34,207 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:37:34,207 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:37:34,208 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:37:35,053 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:35,054 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:37:35,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:35,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:36,054 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:37,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:37,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:38,163 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:39,503 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:39,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:40,055 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:41,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:41,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:43,190 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:43,576 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:43,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:44,056 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:45,622 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:45,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:47,664 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:47,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:48,057 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:48,210 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:48,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:37:48,905 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:37:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:37:49,701 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:49,715 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:51,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:51,758 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:52,058 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:53,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:53,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:54,030 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:54,059 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:55,827 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:55,838 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:57,863 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:57,874 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:58,060 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:37:59,061 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:37:59,899 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:37:59,912 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:01,935 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:01,955 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:02,061 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:03,879 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:38:03,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:38:03,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:38:03,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:38:04,092 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:04,366 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:38:04,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:04,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:06,063 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:06,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:06,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:08,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:08,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:09,137 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:10,064 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:10,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:10,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:12,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:12,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:14,065 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:14,161 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:14,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:14,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:16,627 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:16,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:18,066 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:18,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:18,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:18,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:38:18,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:38:18,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:38:20,114 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:20,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:20,723 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:22,068 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:22,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:22,761 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:24,787 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:24,807 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:25,253 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:26,069 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:26,831 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:26,851 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:28,874 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:28,887 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:30,070 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:30,274 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:30,911 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:30,924 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:32,947 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:32,964 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:33,881 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:38:33,905 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:38:33,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:38:33,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:38:34,071 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:35,356 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:38:35,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:35,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:36,078 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:37,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:37,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:38,072 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:39,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:39,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:41,353 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:41,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:41,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:42,074 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:43,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:43,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:45,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:45,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:46,075 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:46,405 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:47,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:47,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:48,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:38:48,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:38:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:38:49,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:49,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:50,076 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:51,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:51,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:52,059 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:53,756 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:53,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:54,077 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:55,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:55,812 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:57,478 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:38:57,833 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:57,850 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:58,078 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:38:59,879 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:38:59,892 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:02,080 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:02,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:02,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:03,474 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:03,882 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:39:03,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:39:03,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:39:03,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:39:04,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:04,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:06,081 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:07,040 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:39:07,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:07,074 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:08,479 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:09,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:09,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:10,082 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:11,138 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:11,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:12,571 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:39:12,572 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:39:12,573 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:39:12,573 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:39:13,083 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:13,083 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:39:13,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:13,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:13,574 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:14,083 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:15,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:15,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:17,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:17,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:18,084 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:18,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:39:18,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:39:18,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:39:19,099 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:19,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:19,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:21,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:21,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:22,086 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:23,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:23,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:24,584 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:25,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:25,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:26,087 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:27,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:27,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:29,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:29,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:29,627 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:30,088 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:31,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:31,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:33,622 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:33,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:33,884 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:39:33,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:39:33,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:39:33,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:39:34,089 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:35,100 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:35,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:35,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:38,072 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:39:38,091 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:38,095 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:38,106 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:40,128 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:40,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:40,685 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:42,092 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:42,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:42,186 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:44,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:44,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:45,741 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:46,093 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:46,247 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:46,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:48,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:48,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:48,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:39:48,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:39:48,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:39:50,094 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:50,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:50,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:51,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:52,361 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:52,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:54,096 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:54,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:54,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:56,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:56,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:56,793 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:39:58,098 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:39:58,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:39:58,495 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:00,523 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:00,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:01,834 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:02,099 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:02,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:02,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:03,886 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:40:03,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:40:03,906 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:40:03,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:40:04,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:04,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:06,101 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:06,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:06,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:06,889 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:09,091 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:40:09,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:09,128 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:10,102 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:11,154 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:11,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:11,895 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:13,187 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:13,199 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:14,103 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:15,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:15,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:16,903 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:17,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:17,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:18,104 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:18,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:40:18,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:40:18,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:40:19,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:19,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:21,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:21,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:21,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:22,105 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:23,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:23,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:25,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:25,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:26,106 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:26,972 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:27,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:27,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:29,521 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:29,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:30,108 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:31,561 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:31,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:32,011 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:33,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:33,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:33,887 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:40:33,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:40:33,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:40:33,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:40:34,109 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:35,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:35,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:37,083 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:37,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:37,689 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:38,110 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:40,102 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:40:40,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:40,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:42,110 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:42,131 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:42,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:42,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:44,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:44,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:46,111 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:46,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:46,243 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:47,178 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:48,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:48,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:48,906 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:40:48,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:40:48,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:40:50,113 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:50,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:50,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:52,192 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:52,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:52,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:53,275 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:40:53,277 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:40:53,277 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:40:53,278 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:40:54,114 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:54,114 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:40:54,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:54,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:55,114 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:56,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:56,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:57,268 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:40:58,115 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:40:58,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:40:58,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:00,498 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:00,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:02,116 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:02,307 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:02,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:02,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:03,889 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:41:03,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:41:03,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:41:03,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:41:04,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:04,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:06,117 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:06,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:06,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:07,359 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:08,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:08,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:10,118 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:11,064 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:41:11,088 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:11,099 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:12,381 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:13,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:13,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:14,119 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:15,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:15,193 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:17,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:17,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:17,443 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:18,121 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:18,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:41:18,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:41:18,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:41:19,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:19,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:21,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:21,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:22,122 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:22,478 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:23,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:23,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:25,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:25,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:26,123 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:27,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:27,434 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:27,504 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:29,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:29,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:31,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:31,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:32,125 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:32,526 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:33,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:33,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:33,891 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:41:33,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:41:33,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:41:33,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:41:35,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:35,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:36,126 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:37,579 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:37,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:37,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:39,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:39,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:40,127 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:42,089 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:41:42,114 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:42,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:42,616 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:44,129 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:44,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:44,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:46,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:46,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:47,648 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:48,129 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:48,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:48,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:48,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:41:48,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:41:48,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:41:50,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:50,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:52,131 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:52,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:52,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:52,718 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:54,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:54,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:56,132 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:41:56,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:56,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:57,723 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:41:58,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:41:58,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:00,133 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:00,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:00,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:02,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:02,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:02,831 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:03,892 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:42:03,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:42:03,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:42:03,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:42:04,134 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:04,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:04,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:06,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:06,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:07,871 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:08,135 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:08,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:08,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:10,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:10,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:12,137 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:13,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:13,094 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:42:13,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:13,132 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:15,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:15,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:16,138 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:17,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:17,207 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:18,126 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:18,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:42:18,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:42:18,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:42:19,239 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:19,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:20,139 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:21,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:21,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:23,187 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:23,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:23,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:24,140 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:25,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:25,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:27,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:27,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:28,141 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:28,238 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:29,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:29,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:31,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:31,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:32,143 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:33,269 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:33,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:33,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:33,893 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:42:33,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:42:33,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:42:33,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:42:34,344 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:42:34,346 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:42:34,346 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:42:34,346 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:42:35,144 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:35,144 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:42:35,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:35,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:36,144 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:37,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:37,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:38,327 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:39,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:39,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:40,146 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:41,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:41,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:43,343 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:44,094 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:42:44,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:44,132 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:44,147 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:46,167 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:46,181 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:48,148 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:48,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:48,224 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:48,363 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:48,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:42:48,907 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:42:48,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:42:50,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:50,267 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:52,149 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:52,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:52,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:53,392 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:42:54,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:54,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:56,150 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:42:56,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:56,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:58,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:58,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:42:58,453 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:00,152 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:00,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:00,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:02,487 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:02,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:03,500 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:03,895 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:43:03,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:43:03,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:43:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:43:04,153 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:04,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:04,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:06,561 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:06,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:08,154 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:08,504 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:08,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:08,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:10,639 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:10,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:12,155 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:12,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:12,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:13,558 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:15,107 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:43:15,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:15,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:16,156 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:17,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:17,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:18,616 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:18,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:43:18,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:43:18,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:43:19,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:19,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:20,158 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:21,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:21,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:23,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:23,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:23,656 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:24,159 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:25,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:25,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:27,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:27,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:28,160 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:28,681 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:29,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:29,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:31,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:31,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:32,161 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:33,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:33,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:33,704 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:33,896 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:43:33,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:43:33,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:43:33,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:43:35,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:35,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:36,163 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:37,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:37,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:38,748 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:39,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:39,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:40,164 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:41,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:41,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:43,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:43,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:43,842 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:44,165 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:46,089 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:43:46,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:46,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:48,166 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:48,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:48,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:48,863 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:43:48,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:43:48,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:43:50,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:50,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:52,168 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:52,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:52,262 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:53,881 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:43:54,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:54,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:56,171 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:43:56,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:56,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:58,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:58,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:43:58,892 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:00,172 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:00,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:00,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:02,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:02,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:03,899 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:44:03,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:03,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:44:03,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:44:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:44:04,173 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:04,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:04,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:06,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:06,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:08,175 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:08,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:08,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:09,024 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:10,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:10,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:12,176 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:12,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:12,645 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:14,042 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:14,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:14,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:15,137 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:44:15,139 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:44:15,139 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:44:15,140 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:44:15,177 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:44:16,177 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:17,075 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:44:17,114 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:17,126 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:18,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:44:18,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:44:18,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:44:19,126 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:19,151 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:19,163 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:20,178 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:21,186 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:21,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:23,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:23,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:24,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:24,179 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:25,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:25,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:27,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:27,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:28,181 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:29,149 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:29,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:29,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:31,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:31,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:32,182 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:33,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:33,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:33,900 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:44:33,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:44:33,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:44:33,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:44:35,080 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:35,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:35,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:36,183 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:37,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:37,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:39,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:39,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:40,184 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:40,220 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:41,595 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:41,611 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:43,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:43,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:44,185 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:45,232 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:45,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:45,695 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:48,129 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:44:48,154 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:48,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:48,186 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:44:48,908 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:44:48,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:44:50,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:50,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:51,066 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:52,188 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:52,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:52,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:54,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:54,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:56,189 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:44:56,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:56,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:56,332 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:44:58,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:44:58,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:00,190 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:00,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:00,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:01,380 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:02,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:02,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:03,902 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:45:03,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:45:03,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:45:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:45:04,191 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:04,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:04,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:06,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:06,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:07,137 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:08,192 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:08,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:08,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:10,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:10,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:12,193 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:12,483 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:12,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:12,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:14,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:14,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:16,195 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:16,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:16,733 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:17,509 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:18,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:45:18,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:45:18,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:45:19,136 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:45:19,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:19,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:20,196 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:21,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:21,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:22,523 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:23,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:23,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:24,197 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:25,275 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:25,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:27,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:27,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:27,538 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:28,198 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:29,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:29,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:31,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:31,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:32,199 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:32,558 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:33,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:33,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:33,903 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:45:33,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:45:33,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:45:33,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:45:34,200 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:35,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:35,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:37,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:37,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:38,201 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:38,404 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:39,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:39,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:41,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:41,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:42,202 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:43,441 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:43,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:43,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:45,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:45,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:46,203 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:47,692 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:47,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:48,475 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:48,908 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:45:48,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:45:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:45:50,116 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:45:50,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:50,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:50,205 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:52,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:52,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:53,516 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:45:53,578 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:45:53,579 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:45:53,579 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:45:53,579 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:45:54,206 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:54,206 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:45:54,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:54,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:56,206 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:56,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:56,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:58,207 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:45:58,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:58,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:45:58,548 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:00,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:00,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:02,208 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:02,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:02,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:03,555 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:03,905 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:46:03,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:46:03,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:46:03,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:46:04,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:04,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:06,209 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:06,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:06,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:08,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:08,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:08,563 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:10,210 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:10,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:10,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:12,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:12,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:13,575 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:14,212 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:14,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:14,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:16,645 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:16,662 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:18,213 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:18,620 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:18,698 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:18,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:18,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:46:18,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:46:18,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:46:21,127 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:46:21,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:21,167 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:22,214 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:23,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:23,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:23,627 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:25,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:25,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:26,215 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:27,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:27,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:28,641 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:29,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:29,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:30,216 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:31,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:31,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:33,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:33,379 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:33,698 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:33,906 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:46:33,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:46:33,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:46:33,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:46:34,217 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:35,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:35,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:37,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:37,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:38,219 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:38,756 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:39,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:39,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:41,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:41,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:42,220 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:43,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:43,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:43,801 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:45,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:45,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:46,221 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:47,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:47,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:48,837 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:48,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:46:48,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:46:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:46:49,684 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:49,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:50,222 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:52,101 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:46:52,130 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:52,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:53,873 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:46:54,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:54,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:54,223 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:46:56,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:56,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:58,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:58,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:46:58,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:00,225 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:00,279 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:00,291 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:02,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:02,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:03,908 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:47:03,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:47:03,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:47:03,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:47:04,068 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:04,226 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:04,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:04,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:06,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:06,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:08,228 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:08,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:08,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:09,999 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:10,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:10,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:12,229 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:12,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:12,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:14,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:14,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:15,029 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:16,230 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:16,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:16,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:18,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:18,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:18,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:47:18,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:47:18,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:47:20,073 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:20,231 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:20,673 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:20,684 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:23,088 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:47:23,119 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:23,139 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:24,233 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:25,091 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:25,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:25,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:27,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:27,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:28,234 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:29,236 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:29,247 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:30,134 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:31,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:31,287 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:32,235 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:33,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:33,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:33,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:47:33,910 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:47:33,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:47:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:47:34,209 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:47:34,210 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:47:34,210 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:47:34,211 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:47:34,236 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:47:35,212 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:35,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:35,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:36,236 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:37,379 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:37,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:39,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:39,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:40,238 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:41,173 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:41,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:41,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:43,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:43,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:44,240 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:45,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:45,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:46,190 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:47,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:47,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:48,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:47:48,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:47:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:47:49,241 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:49,608 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:49,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:51,285 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:51,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:51,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:53,242 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:54,068 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:47:54,095 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:54,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:56,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:56,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:56,337 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:47:57,244 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:47:58,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:47:58,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:00,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:00,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:01,245 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:01,375 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:02,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:02,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:48:03,911 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:48:03,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:48:03,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:48:04,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:04,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:05,246 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:06,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:06,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:06,417 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:08,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:08,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:09,247 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:10,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:10,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:12,415 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:12,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:12,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:13,248 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:14,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:14,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:16,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:16,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:17,250 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:17,456 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:18,576 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:18,586 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:18,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:48:18,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:48:18,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:48:20,615 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:20,627 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:21,251 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:22,504 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:22,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:22,662 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:25,065 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:48:25,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:25,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:25,252 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:27,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:27,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:27,507 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:29,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:29,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:29,253 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:31,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:31,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:32,525 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:33,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:33,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:33,254 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:33,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:48:33,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:48:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:48:34,072 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:48:35,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:35,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:37,255 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:37,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:37,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:37,532 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:39,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:39,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:41,256 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:41,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:41,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:42,566 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:43,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:43,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:45,258 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:45,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:45,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:47,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:47,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:47,580 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:48,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:48:48,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:48:48,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:48:49,259 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:49,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:49,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:51,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:51,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:52,590 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:53,260 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:53,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:53,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:56,008 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:48:56,032 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:56,049 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:57,261 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:48:57,591 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:48:58,076 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:48:58,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:00,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:00,124 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:01,262 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:02,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:02,168 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:02,608 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:03,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:49:03,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:49:03,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:49:04,088 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:49:04,306 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:04,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:05,264 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:06,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:06,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:07,682 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:08,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:08,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:09,265 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:10,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:10,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:12,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:12,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:12,693 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:13,266 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:14,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:14,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:14,746 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:49:14,748 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:49:14,748 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:49:14,749 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:49:15,267 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:49:16,267 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:16,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:16,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:17,267 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:17,750 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:18,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:18,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:18,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:49:18,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:49:18,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:49:20,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:20,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:21,269 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:22,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:22,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:23,715 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:24,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:24,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:25,270 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:27,142 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:49:27,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:27,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:28,730 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:29,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:29,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:29,271 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:31,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:31,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:33,272 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:33,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:33,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:33,773 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:33,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:49:33,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:49:33,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:49:34,053 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:49:35,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:35,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:37,274 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:37,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:37,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:38,794 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:39,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:39,405 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:41,275 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:41,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:41,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:43,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:43,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:43,820 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:45,276 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:45,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:45,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:47,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:47,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:48,840 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:48,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:49:48,910 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:49:48,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:49:49,278 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:49,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:49,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:51,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:51,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:53,279 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:53,668 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:53,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:53,878 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:49:55,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:55,722 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:57,280 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:49:58,134 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:49:58,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:58,168 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:49:58,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:00,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:00,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:01,281 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:02,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:02,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:03,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:50:03,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:50:03,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:50:04,048 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:50:04,048 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:04,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:04,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:05,283 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:06,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:06,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:08,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:08,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:09,284 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:09,995 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:10,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:10,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:12,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:12,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:13,285 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:14,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:14,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:15,037 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:16,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:16,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:17,286 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:18,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:18,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:18,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:50:18,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:50:18,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:50:20,098 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:20,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:20,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:21,288 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:22,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:22,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:24,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:24,677 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:25,116 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:25,289 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:26,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:26,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:29,121 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:50:29,151 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:29,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:29,290 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:30,124 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:31,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:31,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:33,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:33,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:33,291 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:33,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:50:33,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:50:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:50:34,137 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:50:35,138 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:35,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:35,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:37,293 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:37,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:37,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:39,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:39,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:40,164 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:41,295 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:41,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:41,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:43,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:43,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:45,209 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:45,296 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:45,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:45,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:47,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:47,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:48,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:50:48,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:50:48,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:50:49,297 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:49,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:49,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:51,135 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:51,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:51,600 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:53,299 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:53,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:53,643 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:55,331 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:50:55,332 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:50:55,332 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:50:55,333 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:50:55,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:55,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:56,300 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:56,300 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:50:56,334 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:50:57,300 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:50:57,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:50:57,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:00,132 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:51:00,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:00,169 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:01,301 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:02,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:02,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:02,330 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:51:03,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:51:03,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:51:04,131 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:51:04,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:04,243 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:05,303 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:06,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:06,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:07,349 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:08,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:08,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:09,304 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:10,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:10,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:12,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:12,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:12,402 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:13,305 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:14,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:14,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:16,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:16,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:17,307 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:17,453 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:18,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:18,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:18,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:51:18,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:51:18,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:51:20,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:20,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:21,308 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:22,461 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:22,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:22,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:24,614 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:24,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:25,309 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:26,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:26,673 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:27,481 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:28,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:28,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:29,310 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:31,124 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:51:31,152 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:31,163 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:32,551 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:33,193 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:33,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:33,311 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:33,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:51:33,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:51:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:51:34,124 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:51:35,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:35,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:37,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:37,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:37,313 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:37,586 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:39,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:39,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:41,314 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:41,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:41,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:42,625 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:43,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:43,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:45,315 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:45,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:45,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:47,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:47,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:47,628 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:48,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:51:48,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:51:48,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:51:49,316 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:49,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:49,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:51,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:51,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:52,697 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:53,318 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:53,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:53,600 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:55,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:55,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:57,319 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:51:57,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:57,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:57,704 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:51:59,710 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:51:59,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:01,320 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:02,133 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:52:02,155 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:02,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:02,751 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:52:03,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:52:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:52:04,089 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:52:04,188 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:04,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:05,321 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:06,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:06,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:07,820 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:08,263 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:08,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:09,323 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:10,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:10,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:12,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:12,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:12,853 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:13,324 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:14,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:14,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:16,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:16,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:17,326 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:17,880 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:18,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:18,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:18,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:52:18,911 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:52:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:52:20,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:20,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:21,327 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:22,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:22,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:22,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:24,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:24,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:25,328 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:26,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:26,635 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:28,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:28,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:28,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:29,330 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:30,695 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:30,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:33,135 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:52:33,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:33,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:33,331 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:33,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:52:33,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:52:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:52:34,099 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:52:34,100 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:35,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:35,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:36,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:52:36,149 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:52:36,149 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:52:36,150 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:52:36,332 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:52:37,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:37,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:37,333 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:38,333 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:39,151 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:39,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:39,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:41,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:41,334 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:41,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:43,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:43,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:44,153 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:45,335 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:45,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:45,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:47,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:47,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:48,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:52:48,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:52:48,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:52:49,337 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:49,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:49,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:50,128 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:51,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:51,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:53,338 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:53,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:53,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:55,183 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:52:55,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:55,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:57,340 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:52:57,628 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:57,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:59,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:52:59,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:00,229 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:01,341 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:01,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:01,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:03,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:53:03,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:53:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:53:04,083 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:53:04,143 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:53:04,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:04,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:05,302 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:05,343 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:06,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:06,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:08,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:08,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:09,344 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:10,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:10,291 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:10,323 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:12,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:12,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:13,345 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:14,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:14,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:15,346 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:16,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:16,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:16,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:18,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:18,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:18,911 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:53:18,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:53:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:53:19,347 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:20,487 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:20,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:22,084 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:22,530 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:22,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:23,349 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:24,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:24,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:26,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:26,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:27,207 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:27,350 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:28,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:28,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:30,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:30,704 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:31,351 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:32,215 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:32,727 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:32,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:33,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:53:33,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:53:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:53:34,140 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:53:35,142 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:53:35,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:35,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:35,353 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:37,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:37,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:37,245 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:39,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:39,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:39,354 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:41,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:41,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:42,248 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:43,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:43,340 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:43,356 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:45,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:45,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:47,302 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:47,357 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:47,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:47,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:53:48,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:53:48,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:53:49,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:49,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:51,358 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:51,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:51,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:52,303 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:53,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:53,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:55,359 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:55,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:55,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:57,323 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:53:57,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:57,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:59,361 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:53:59,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:53:59,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:01,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:01,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:02,327 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:03,362 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:03,708 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:03,727 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:03,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:54:03,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:54:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:54:04,086 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:54:06,129 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:54:06,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:06,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:07,363 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:07,364 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:08,224 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:08,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:10,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:10,279 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:11,365 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:12,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:12,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:12,366 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:14,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:14,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:14,450 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:54:14,452 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:54:14,452 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:54:14,453 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:54:15,367 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:15,368 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:54:16,367 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:16,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:16,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:17,454 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:18,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:18,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:18,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:54:18,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:54:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:54:19,369 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:20,473 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:20,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:22,462 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:22,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:22,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:23,370 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:24,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:24,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:26,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:26,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:27,478 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:28,633 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:28,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:29,372 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:30,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:30,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:32,511 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:32,714 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:32,729 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:33,373 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:33,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:54:33,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:54:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:54:34,196 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:54:34,764 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:34,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:37,174 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:54:37,207 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:37,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:37,375 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:37,551 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:39,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:39,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:41,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:41,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:41,376 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:42,561 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:43,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:43,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:45,377 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:45,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:45,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:47,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:47,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:47,678 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:54:48,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:54:48,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:54:49,379 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:49,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:49,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:51,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:51,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:52,708 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:53,380 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:53,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:53,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:55,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:55,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:57,382 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:54:57,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:57,639 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:57,715 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:54:59,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:54:59,681 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:01,383 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:01,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:01,723 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:02,745 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:03,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:03,763 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:03,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:55:03,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:55:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:55:04,042 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:55:05,384 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:05,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:05,803 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:08,203 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:55:08,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:08,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:08,746 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:09,386 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:10,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:10,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:12,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:12,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:13,387 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:13,772 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:14,361 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:14,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:16,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:16,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:17,388 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:18,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:18,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:18,777 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:18,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:55:18,912 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:55:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:55:20,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:20,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:21,390 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:22,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:22,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:23,800 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:24,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:24,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:25,391 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:26,592 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:26,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:28,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:28,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:28,837 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:29,393 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:30,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:30,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:32,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:32,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:33,394 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:33,887 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:33,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:55:33,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:55:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:55:34,073 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:55:34,751 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:34,769 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:36,794 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:36,805 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:37,396 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:38,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:39,214 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:55:39,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:39,252 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:41,275 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:41,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:41,397 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:43,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:43,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:43,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:45,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:45,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:45,399 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:47,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:47,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:48,912 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:55:48,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:55:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:55:49,132 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:49,401 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:49,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:49,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:51,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:51,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:53,402 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:53,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:53,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:55,006 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:55:55,070 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 19:55:55,072 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 19:55:55,072 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 19:55:55,074 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 19:55:55,403 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 19:55:55,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:55,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:56,403 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:57,403 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:55:57,572 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:57,592 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:59,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:55:59,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:00,076 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:01,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:01,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:03,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:03,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:03,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:56:03,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:56:03,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:56:04,104 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:56:05,105 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:05,729 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:05,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:07,774 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:07,785 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:10,106 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:10,194 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:56:10,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:10,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:12,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:12,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:14,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:14,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:15,106 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:16,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:16,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:17,410 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:56:18,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:18,386 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:18,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:56:18,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:56:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:56:20,146 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:20,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:20,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:22,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:22,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:24,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:24,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:25,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:26,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:26,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:28,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:28,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:30,148 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:30,630 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:30,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:32,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:32,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:33,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:56:33,913 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:56:33,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:56:34,113 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:56:34,713 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:34,728 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:36,089 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:36,755 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:36,767 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:37,417 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:56:38,790 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:38,802 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:41,090 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:41,202 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:56:41,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:41,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:43,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:43,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:45,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:45,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:46,092 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:47,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:47,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:48,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:56:48,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:56:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:56:49,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:49,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:51,118 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:51,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:51,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:53,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:53,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:55,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:55,495 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:57,025 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:56:57,423 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:56:57,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:57,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:59,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:56:59,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:01,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:01,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:02,026 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:03,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:03,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:03,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:57:03,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:57:03,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:57:04,108 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:57:05,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:05,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:07,109 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:07,730 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:07,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:09,771 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:09,785 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:12,110 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:12,184 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:57:12,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:12,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:14,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:14,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:16,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:16,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:17,429 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:57:17,958 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:18,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:18,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:18,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:57:18,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:57:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:57:20,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:20,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:22,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:22,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:23,145 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:24,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:24,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:26,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:26,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:28,146 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:28,523 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:28,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:30,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:30,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:32,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:32,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:33,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:33,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:57:33,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:57:33,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:57:34,071 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:57:34,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:34,666 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:36,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:36,707 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:37,435 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:57:38,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:38,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:38,898 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:40,781 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:40,793 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:43,211 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:57:43,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:43,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:43,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:45,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:45,285 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:47,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:47,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:47,439 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:57:48,913 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:57:48,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:57:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:57:49,141 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:49,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:49,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:51,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:51,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:53,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:53,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:54,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:55,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:55,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:57,443 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:57:57,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:57,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:59,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:57:59,545 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:57:59,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:01,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:01,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:03,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:03,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:03,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:58:03,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:58:03,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:58:04,141 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:58:05,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:05,657 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:05,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:07,446 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:58:07,697 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:07,708 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:09,735 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:09,746 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:10,844 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:11,778 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:11,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:14,221 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:58:14,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:14,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:15,845 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:16,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:16,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:17,449 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:58:18,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:18,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:18,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:58:18,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:58:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:58:20,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:20,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:21,076 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:22,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:22,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:24,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:24,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:26,077 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:26,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:26,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:28,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:28,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:29,453 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:58:30,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:30,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:31,294 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:32,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:32,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:58:33,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:58:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:58:34,089 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:58:34,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:34,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:36,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:36,701 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:37,091 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:38,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:38,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:39,456 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:58:40,785 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:40,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:42,338 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:42,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:42,842 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:45,261 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:58:45,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:45,297 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:47,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:47,340 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:47,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:48,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:58:48,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:58:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:58:49,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:49,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:49,459 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:58:51,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:51,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:53,070 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:53,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:53,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:55,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:55,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:57,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:57,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:58,070 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:58:59,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:58:59,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:01,463 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:59:01,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:01,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:03,541 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:03,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:03,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:03,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:59:03,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:59:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:59:04,055 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:59:05,662 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:05,678 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:07,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:07,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:09,057 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:09,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:09,753 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:11,781 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:11,792 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:13,821 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:13,833 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:14,058 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:16,237 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:59:16,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:16,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:18,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:18,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:18,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:59:18,914 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:59:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:59:19,090 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:20,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:20,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:21,469 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:59:22,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:22,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:24,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:24,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:24,517 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:26,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:26,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:28,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:28,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:29,518 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:30,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:30,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:31,472 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:59:32,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:32,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:59:33,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:59:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:59:34,083 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 19:59:34,627 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:34,649 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:35,084 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:36,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:36,683 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:38,711 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:38,722 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:40,084 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:40,754 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:40,774 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:42,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:42,815 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:43,476 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:59:44,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:44,856 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:45,362 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:47,258 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 19:59:47,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:47,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:48,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 19:59:48,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 19:59:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 19:59:49,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:49,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:51,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:51,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:51,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:53,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:53,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:53,479 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 19:59:55,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:55,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:56,288 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 19:59:57,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:57,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:59,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 19:59:59,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:01,288 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:01,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:01,565 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:03,482 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:00:03,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:03,611 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:03,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:00:03,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:00:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:00:04,122 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:00:05,633 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:05,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:07,124 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:07,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:07,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:09,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:09,720 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:11,742 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:11,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:12,125 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:13,486 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:00:13,786 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:13,799 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:15,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:15,840 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:17,886 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:18,254 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:00:18,275 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:18,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:18,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:00:18,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:00:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:00:20,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:20,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:22,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:22,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:23,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:24,380 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:24,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:26,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:26,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:27,490 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:00:28,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:28,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:29,066 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:30,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:30,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:32,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:32,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:00:33,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:00:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:00:34,042 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:00:34,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:34,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:35,043 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:36,592 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:36,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:38,627 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:38,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:40,043 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:40,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:40,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:41,495 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:00:42,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:42,711 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:44,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:44,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:45,261 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:46,783 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:46,798 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:48,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:00:48,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:00:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:00:49,210 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:00:49,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:49,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:51,078 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:51,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:51,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:53,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:53,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:53,499 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:00:55,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:55,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:56,892 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:00:57,379 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:57,390 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:59,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:00:59,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:01,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:01,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:01,893 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:03,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:03,502 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:01:03,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:01:03,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:01:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:01:04,083 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:01:05,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:05,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:07,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:07,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:07,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:09,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:09,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:11,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:11,661 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:12,086 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:13,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:13,704 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:15,726 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:15,738 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:17,086 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:17,762 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:17,778 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:01:18,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:01:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:01:20,179 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:01:20,202 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:20,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:22,140 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:22,236 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:22,248 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:23,508 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:01:24,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:24,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:26,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:26,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:27,335 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:28,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:28,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:30,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:30,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:32,335 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:32,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:32,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:01:33,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:01:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:01:34,051 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:01:34,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:34,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:35,512 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:01:36,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:36,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:38,334 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:38,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:38,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:40,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:40,601 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:42,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:42,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:43,334 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:44,662 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:44,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:46,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:46,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:47,516 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:01:48,575 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:48,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:48,761 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:48,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:01:48,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:01:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:01:51,169 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:01:51,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:51,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:53,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:53,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:54,062 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:01:55,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:55,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:57,306 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:57,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:57,519 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:01:59,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:59,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:01:59,752 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:01,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:01,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:03,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:03,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:02:03,915 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:02:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:02:04,120 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:02:05,121 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:05,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:05,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:07,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:07,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:09,522 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:02:09,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:09,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:10,671 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:11,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:11,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:13,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:13,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:15,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:15,664 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:15,671 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:17,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:17,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:18,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:02:18,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:02:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:02:19,526 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:02:19,725 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:19,737 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:21,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:22,162 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:02:22,188 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:22,199 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:24,221 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:24,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:26,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:26,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:26,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:28,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:28,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:30,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:30,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:31,530 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:02:31,581 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:32,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:32,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:33,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:02:33,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:02:33,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:02:34,116 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:02:34,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:34,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:36,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:36,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:37,117 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:38,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:38,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:40,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:40,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:41,533 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:02:42,384 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:42,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:42,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:44,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:44,614 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:46,643 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:46,655 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:47,385 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:48,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:48,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:02:48,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:02:48,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:02:50,725 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:50,736 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:53,136 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:02:53,138 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:53,164 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:53,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:55,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:55,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:57,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:57,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:58,138 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:02:59,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:02:59,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:01,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:01,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:01,540 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:03:03,358 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:03,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:03,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:03,915 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:03:03,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:03:03,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:03:04,076 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:03:05,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:05,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:07,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:07,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:09,078 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:09,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:09,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:11,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:11,544 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:03:11,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:13,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:13,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:14,598 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:15,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:15,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:17,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:17,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:03:18,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:03:18,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:03:19,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:19,710 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:20,075 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:21,734 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:21,748 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:24,144 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:03:24,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:24,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:25,075 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:26,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:26,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:28,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:28,269 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:30,076 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:30,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:30,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:31,550 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:03:32,335 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:32,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:33,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:03:33,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:03:33,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:03:34,051 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:03:34,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:34,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:36,052 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:36,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:36,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:38,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:38,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:40,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:40,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:41,053 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:42,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:42,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:43,554 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:03:44,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:44,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:46,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:46,614 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:46,947 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:48,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:48,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:03:48,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:03:48,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:03:50,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:50,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:52,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:52,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:52,726 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:53,557 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:03:55,133 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:03:55,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:55,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:57,204 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:57,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:57,679 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:03:59,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:03:59,262 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:01,285 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:01,297 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:02,680 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:03,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:03,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:03,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:04:03,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:04:03,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:04:04,045 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:04:05,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:05,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:05,561 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:04:07,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:07,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:07,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:09,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:09,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:11,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:11,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:12,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:13,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:13,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:15,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:15,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:17,565 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:04:17,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:17,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:18,283 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:04:18,916 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:04:18,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:04:19,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:19,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:21,683 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:21,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:23,717 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:23,728 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:24,081 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:26,122 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:04:26,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:26,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:28,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:28,221 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:28,568 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:04:29,829 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:30,248 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:30,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:32,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:32,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:33,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:04:33,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:04:33,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:04:34,069 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:04:34,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:34,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:35,070 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:36,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:36,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:38,405 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:38,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:38,572 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:04:40,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:40,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:41,010 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:42,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:42,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:44,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:44,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:46,011 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:46,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:46,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:48,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:48,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:48,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:04:48,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:04:48,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:04:50,576 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:04:50,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:50,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:51,136 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:52,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:52,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:54,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:54,717 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:56,137 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:04:57,111 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:04:57,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:57,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:59,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:04:59,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:00,580 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:05:01,182 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:01,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:01,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:03,279 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:03,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:05:03,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:05:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:05:04,141 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:05:05,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:05,335 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:07,143 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:07,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:07,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:09,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:09,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:11,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:11,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:12,583 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:05:13,121 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:13,474 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:13,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:15,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:15,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:17,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:17,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:18,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:18,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:05:18,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:05:18,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:05:19,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:19,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:21,630 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:21,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:23,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:23,663 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:23,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:25,699 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:25,711 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:28,096 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:05:28,118 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:28,123 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:28,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:30,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:30,167 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:32,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:32,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:32,589 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:05:33,916 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:05:33,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:05:33,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:05:34,128 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:05:34,129 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:34,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:34,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:36,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:36,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:38,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:38,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:39,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:40,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:40,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:42,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:42,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:42,591 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:05:44,392 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:44,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:44,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:46,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:46,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:48,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:48,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:05:48,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:05:48,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:05:50,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:50,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:50,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:52,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:52,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:52,594 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:05:54,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:54,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:55,505 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:05:56,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:56,657 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:59,047 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:05:59,072 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:05:59,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:00,505 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:01,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:01,130 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:03,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:03,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:06:03,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:06:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:06:04,093 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:06:04,597 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:06:05,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:05,209 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:06,094 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:07,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:07,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:09,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:09,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:11,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:11,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:11,869 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:12,599 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:06:13,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:13,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:15,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:15,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:16,870 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:17,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:17,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:06:18,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:06:18,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:06:19,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:19,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:21,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:21,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:22,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:22,602 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:06:23,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:23,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:25,603 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:25,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:27,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:27,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:27,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:30,047 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:06:30,072 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:30,086 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:32,108 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:32,112 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:32,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:06:33,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:06:33,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:06:34,044 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:06:34,147 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:34,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:36,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:36,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:38,045 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:38,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:38,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:40,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:40,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:42,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:42,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:42,607 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:06:43,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:44,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:44,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:46,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:46,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:48,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:48,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:48,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:06:48,917 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:06:48,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:06:50,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:50,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:52,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:52,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:53,121 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:54,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:54,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:56,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:56,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:58,121 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:06:58,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:06:58,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:00,991 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:07:01,018 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:01,029 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:02,612 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:07:03,065 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:03,079 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:07:03,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:07:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:07:04,088 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:07:04,088 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:05,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:05,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:07,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:07,168 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:09,089 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:09,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:09,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:11,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:11,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:13,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:13,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:14,090 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:15,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:15,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:17,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:17,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:18,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:07:18,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:07:18,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:07:19,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:19,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:20,060 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:21,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:21,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:22,617 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:07:23,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:23,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:25,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:25,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:26,040 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:27,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:27,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:29,580 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:29,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:31,041 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:31,991 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:07:32,017 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:32,032 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:33,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:07:33,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:07:33,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:07:34,057 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:34,077 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:34,077 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:07:34,620 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:07:36,078 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:36,102 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:36,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:38,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:38,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:40,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:40,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:41,079 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:42,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:42,225 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:44,247 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:44,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:46,287 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:46,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:46,623 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:07:46,846 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:48,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:48,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:48,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:07:48,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:07:48,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:07:50,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:50,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:52,144 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:52,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:52,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:54,434 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:54,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:56,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:56,487 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:56,626 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:07:57,270 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:07:58,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:07:58,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:00,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:00,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:02,270 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:02,947 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:08:02,973 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:02,988 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:03,917 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:08:03,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:08:03,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:08:04,092 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:08:05,018 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:05,031 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:06,628 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:07,060 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:07,071 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:07,700 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:09,101 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:09,112 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:11,141 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:11,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:12,701 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:13,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:13,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:15,224 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:15,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:16,631 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:17,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:17,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:18,319 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:18,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:08:18,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:08:18,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:08:19,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:19,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:21,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:21,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:23,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:23,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:24,067 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:25,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:25,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:27,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:27,476 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:28,635 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:29,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:29,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:29,875 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:31,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:31,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:33,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:08:33,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:08:33,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:08:33,945 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:08:33,970 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:33,982 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:34,135 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:08:35,136 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:36,011 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:36,025 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:38,050 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:38,062 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:38,640 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:40,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:40,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:40,372 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:41,479 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:08:41,481 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send():382] send: metric +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:08:41,482 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:08:41,483 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:08:41,641 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:08:42,126 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:42,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:42,641 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:43,642 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:44,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:44,176 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:44,642 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:45,423 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:46,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:46,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:46,643 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:48,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:48,251 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:48,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:08:48,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:08:48,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:08:50,275 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:50,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:50,432 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:50,644 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:50,644 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/config.yaml +2024-01-22 20:08:52,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:52,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:54,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:54,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:54,645 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:08:56,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:56,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:56,464 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:08:58,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:58,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:08:58,646 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:00,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:00,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:01,480 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:02,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:02,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:02,647 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:03,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:09:03,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:09:03,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:09:04,109 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:09:04,921 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:09:04,946 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:04,957 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:06,494 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:06,648 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:06,991 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:07,002 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:09,028 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:09,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:11,072 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:11,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:11,544 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:12,650 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:13,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:13,136 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:15,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:15,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:16,552 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:16,651 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:17,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:17,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:18,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:09:18,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:09:18,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:09:19,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:19,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:20,652 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:21,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:21,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:21,588 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:23,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:23,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:24,653 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:25,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:25,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:26,595 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:27,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:27,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:28,654 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:29,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:29,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:31,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:31,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:31,628 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:32,655 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:33,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:33,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:33,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:09:33,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:09:33,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:09:34,150 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:09:35,944 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:09:35,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:35,989 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:36,656 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:37,152 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:38,018 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:38,030 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:40,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:40,081 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:40,657 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:42,105 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:42,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:42,660 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:44,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:44,151 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:44,659 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:46,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:46,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:47,674 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:48,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:48,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:48,660 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:48,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:09:48,918 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:09:48,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:09:50,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:50,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:52,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:52,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:52,661 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:52,715 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:54,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:54,345 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:56,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:56,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:56,662 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:09:57,748 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:09:58,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:09:58,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:00,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:00,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:00,663 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:02,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:02,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:02,764 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:03,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:10:03,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:10:03,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:10:04,116 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:10:04,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:04,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:04,665 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:06,927 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:10:06,952 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:06,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:07,777 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:08,666 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:08,992 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:09,006 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:11,034 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:11,051 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:12,667 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:12,793 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:13,076 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:13,093 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:15,119 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:15,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:16,668 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:17,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:17,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:17,797 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:18,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:10:18,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:10:18,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:10:19,199 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:19,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:20,669 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:21,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:21,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:21,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:10:21,908 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:10:21,909 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:10:21,909 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:10:22,669 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:10:22,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:23,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:23,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:23,670 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:24,670 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:25,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:25,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:27,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:27,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:28,671 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:28,877 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:29,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:29,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:31,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:31,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:32,672 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:33,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:33,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:33,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:33,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:10:33,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:10:33,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:10:34,114 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:10:35,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:35,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:36,673 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:37,917 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:10:37,947 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:37,960 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:38,970 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:39,992 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:40,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:40,674 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:42,040 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:42,054 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:43,987 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:44,080 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:44,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:44,676 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:46,124 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:46,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:48,166 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:48,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:48,677 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:48,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:10:48,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:10:48,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:10:49,059 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:50,204 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:50,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:52,247 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:52,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:52,678 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:54,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:54,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:55,035 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:10:56,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:56,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:56,679 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:10:58,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:10:58,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:00,065 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:00,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:00,422 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:00,680 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:02,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:02,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:11:03,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:11:03,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:11:04,100 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:11:04,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:04,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:04,681 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:05,101 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:06,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:06,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:08,683 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:08,921 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:11:08,948 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:08,959 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:10,125 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:10,985 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:10,996 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:12,684 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:13,025 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:13,037 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:15,066 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:15,079 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:15,162 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:16,685 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:17,104 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:17,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:18,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:11:18,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:11:18,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:11:19,141 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:19,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:20,686 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:21,052 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:21,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:21,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:23,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:23,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:24,687 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:25,263 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:25,279 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:26,213 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:27,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:27,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:28,689 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:29,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:29,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:31,290 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:31,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:31,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:32,690 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:33,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:33,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:33,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:11:33,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:11:33,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:11:34,071 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:11:35,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:35,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:36,330 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:36,691 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:37,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:37,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:39,924 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:11:39,962 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:39,984 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:40,693 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:41,344 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:42,012 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:42,024 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:44,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:44,067 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:44,694 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:46,092 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:46,108 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:46,366 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:48,132 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:48,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:48,695 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:48,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:11:48,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:11:48,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:11:50,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:50,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:51,427 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:52,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:52,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:52,696 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:54,247 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:54,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:56,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:56,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:56,482 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:11:56,697 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:11:58,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:11:58,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:00,361 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:00,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:00,698 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:01,540 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:02,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:02,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:02,602 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:12:02,603 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:12:02,604 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:12:02,604 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:12:02,699 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:12:03,699 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:12:03,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:12:03,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:12:04,135 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:12:04,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:04,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:04,699 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:06,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:06,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:06,604 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:08,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:08,523 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:08,701 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:10,927 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:12:10,952 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:10,969 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:11,654 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:12,702 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:12,995 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:13,006 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:15,030 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:15,041 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:16,677 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:16,703 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:17,073 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:17,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:18,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:12:18,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:12:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:12:19,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:19,136 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:20,704 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:21,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:21,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:22,071 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:23,202 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:23,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:24,705 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:25,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:25,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:27,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:27,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:27,742 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:28,706 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:29,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:29,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:31,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:31,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:32,708 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:32,800 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:33,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:33,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:33,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:12:33,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:12:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:12:34,049 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:12:35,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:35,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:36,709 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:37,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:37,500 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:37,844 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:39,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:39,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:40,710 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:41,924 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:12:41,950 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:41,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:42,863 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:43,988 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:43,999 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:44,712 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:46,031 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:46,044 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:47,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:48,074 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:48,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:48,713 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:48,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:12:48,919 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:12:48,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:12:50,117 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:50,131 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:52,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:52,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:52,714 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:52,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:54,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:54,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:56,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:56,248 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:56,715 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:12:57,958 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:12:58,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:12:58,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:00,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:00,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:00,716 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:02,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:02,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:02,989 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:03,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:13:03,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:13:03,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:13:04,137 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:13:04,386 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:04,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:04,717 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:06,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:06,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:08,041 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:08,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:08,474 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:08,719 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:10,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:10,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:12,720 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:12,901 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:13:12,922 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:12,934 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:13,060 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:14,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:14,977 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:16,721 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:17,005 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:17,018 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:18,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:18,919 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:13:18,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:13:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:13:19,044 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:19,062 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:20,722 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:21,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:21,110 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:23,084 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:23,135 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:23,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:24,723 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:25,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:25,187 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:27,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:27,221 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:28,120 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:28,725 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:29,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:29,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:31,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:31,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:32,726 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:33,135 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:33,322 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:33,340 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:33,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:13:33,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:13:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:13:34,120 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:13:35,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:35,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:36,727 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:37,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:37,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:38,150 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:39,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:39,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:40,728 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:41,007 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:13:41,009 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:13:41,009 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:13:41,010 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:13:41,495 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:41,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:41,728 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:41,728 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:13:42,728 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:43,892 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:13:43,917 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:43,929 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:44,010 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:45,955 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:45,969 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:46,730 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:47,997 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:48,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:48,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:13:48,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:13:48,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:13:49,139 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:50,039 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:50,051 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:50,731 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:52,077 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:52,092 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:54,118 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:54,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:54,172 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:13:54,732 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:56,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:56,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:58,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:58,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:13:58,733 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:13:59,184 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:00,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:00,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:02,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:02,279 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:02,734 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:03,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:14:03,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:14:03,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:14:04,150 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:14:04,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:04,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:05,151 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:06,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:06,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:06,736 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:08,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:08,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:10,255 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:10,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:10,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:10,737 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:12,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:12,495 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:14,738 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:14,886 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:14:14,919 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:14,934 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:16,255 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:16,958 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:16,971 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:18,739 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:18,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:14:18,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:14:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:14:19,001 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:19,013 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:21,047 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:21,059 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:21,287 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:22,740 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:23,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:23,099 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:25,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:25,141 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:26,307 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:26,742 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:27,168 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:27,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:29,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:29,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:30,743 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:31,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:31,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:31,309 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:33,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:33,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:33,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:14:33,921 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:14:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:14:34,136 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:14:34,744 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:35,338 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:35,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:37,138 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:37,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:37,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:38,745 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:39,420 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:39,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:41,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:41,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:42,415 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:42,746 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:43,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:43,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:45,905 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:14:45,933 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:45,944 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:46,747 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:47,969 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:47,988 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:48,413 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:48,920 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:14:48,920 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:14:48,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:14:50,013 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:50,027 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:50,749 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:52,052 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:52,065 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:53,430 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:54,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:54,108 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:54,750 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:14:56,135 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:56,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:58,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:58,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:14:58,480 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:14:58,751 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:00,220 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:00,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:02,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:02,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:02,752 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:03,512 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:03,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:15:03,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:15:03,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:15:04,143 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:15:04,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:04,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:06,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:06,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:06,754 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:08,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:08,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:08,535 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:10,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:10,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:10,755 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:12,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:12,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:13,555 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:14,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:14,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:14,756 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:16,937 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:15:16,966 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:16,977 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:18,757 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:18,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:15:18,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:15:18,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:15:19,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:19,021 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:19,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:21,052 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:21,064 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:21,760 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:15:21,762 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:15:21,762 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:15:21,763 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:15:22,758 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:22,759 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:15:23,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:23,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:23,759 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:24,763 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:25,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:25,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:26,760 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:27,163 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:27,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:29,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:29,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:30,666 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:30,761 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:31,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:31,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:33,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:33,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:33,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:15:33,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:15:33,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:15:34,143 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:15:34,762 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:35,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:35,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:36,144 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:37,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:37,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:38,763 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:39,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:39,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:41,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:41,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:41,677 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:42,764 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:43,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:43,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:45,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:45,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:46,701 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:46,765 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:47,967 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:15:47,996 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:48,015 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:48,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:15:48,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:15:48,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:15:50,041 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:50,060 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:50,766 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:51,737 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:52,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:52,098 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:54,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:54,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:54,767 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:15:56,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:56,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:56,754 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:15:58,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:58,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:15:58,768 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:00,239 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:00,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:01,772 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:02,280 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:02,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:02,769 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:03,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:16:03,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:16:03,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:16:04,095 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:16:04,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:04,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:06,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:06,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:06,771 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:06,824 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:08,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:08,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:10,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:10,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:10,772 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:11,846 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:12,480 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:12,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:14,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:14,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:14,773 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:16,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:16,583 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:16,847 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:18,774 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:18,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:16:18,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:16:18,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:16:18,988 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:16:19,020 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:19,034 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:21,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:21,081 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:21,901 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:22,775 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:23,106 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:23,122 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:25,147 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:25,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:26,776 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:26,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:27,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:27,202 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:29,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:29,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:30,777 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:31,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:31,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:32,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:33,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:33,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:33,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:16:33,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:16:33,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:16:34,104 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:16:34,779 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:35,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:35,361 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:37,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:37,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:37,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:38,780 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:39,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:39,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:41,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:41,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:42,781 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:42,958 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:43,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:43,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:45,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:45,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:46,782 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:47,586 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:47,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:47,974 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:48,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:16:48,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:16:48,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:16:49,989 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:16:50,019 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:50,038 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:50,783 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:52,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:52,081 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:53,010 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:54,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:54,120 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:54,784 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:16:56,145 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:56,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:58,024 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:16:58,187 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:58,204 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:16:58,785 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:00,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:00,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:02,118 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:17:02,120 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:17:02,120 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:17:02,121 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:17:02,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:02,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:02,786 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:02,786 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:17:03,122 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:03,787 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:03,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:17:03,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:17:03,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:17:04,161 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:17:04,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:04,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:06,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:06,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:06,787 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:08,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:08,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:09,088 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:10,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:10,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:10,788 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:12,473 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:12,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:14,105 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:14,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:14,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:14,789 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:16,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:16,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:18,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:18,608 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:18,791 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:18,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:17:18,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:17:18,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:17:19,127 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:21,006 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:17:21,039 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:21,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:23,080 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:23,093 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:24,206 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:24,792 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:25,128 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:25,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:27,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:27,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:28,793 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:29,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:29,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:30,192 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:31,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:31,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:32,795 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:33,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:33,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:33,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:17:33,922 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:17:33,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:17:34,136 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:17:35,203 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:35,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:35,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:36,796 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:37,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:37,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:38,796 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:39,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:39,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:40,242 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:41,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:41,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:42,798 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:43,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:43,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:45,281 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:45,552 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:45,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:46,799 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:47,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:47,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:48,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:17:48,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:17:48,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:17:49,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:49,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:50,329 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:50,800 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:52,034 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:17:52,064 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:52,083 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:54,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:54,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:54,801 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:17:55,378 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:17:56,147 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:56,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:58,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:58,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:17:58,802 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:00,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:00,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:00,406 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:02,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:02,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:02,803 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:03,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:18:03,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:18:03,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:18:04,146 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:18:04,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:04,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:06,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:06,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:06,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:06,805 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:08,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:08,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:10,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:10,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:10,806 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:11,406 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:12,462 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:12,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:14,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:14,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:14,807 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:16,413 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:16,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:16,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:18,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:18,808 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:18,907 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:18,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:18:18,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:18:18,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:18:20,945 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:20,956 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:22,298 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:23,749 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:18:23,779 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:23,791 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:23,809 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:25,816 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:25,998 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:27,411 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:27,811 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:28,027 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:28,056 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:30,082 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:30,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:31,812 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:32,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:32,429 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:32,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:33,922 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:18:33,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:18:33,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:18:34,131 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:18:34,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:34,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:35,813 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:36,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:36,639 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:38,418 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:38,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:38,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:39,814 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:40,714 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:41,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:42,474 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:18:42,475 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:18:42,476 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:18:42,476 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:18:42,815 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:18:43,139 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:43,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:43,476 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:43,815 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:44,816 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:45,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:45,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:47,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:47,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:48,817 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:48,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:18:48,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:18:48,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:18:49,118 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:49,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:49,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:51,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:51,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:52,818 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:53,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:53,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:54,501 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:55,840 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:18:55,864 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:55,875 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:56,819 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:18:57,899 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:57,916 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:59,615 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:18:59,939 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:18:59,951 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:00,820 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:01,973 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:01,985 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:03,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:19:03,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:19:03,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:19:04,007 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:04,021 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:04,433 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:19:04,821 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:05,434 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:06,050 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:06,061 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:08,088 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:08,099 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:08,823 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:10,132 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:10,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:10,631 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:12,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:12,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:12,823 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:14,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:14,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:15,636 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:16,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:16,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:16,825 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:18,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:18,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:18,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:19:18,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:19:18,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:19:20,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:20,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:20,826 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:21,140 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:22,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:22,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:24,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:24,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:24,827 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:26,628 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:26,890 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:19:26,914 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:26,926 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:28,829 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:28,959 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:28,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:30,994 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:31,007 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:31,651 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:32,830 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:33,033 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:33,049 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:33,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:19:33,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:19:33,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:19:34,132 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:19:35,070 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:35,082 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:36,674 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:36,831 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:37,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:37,119 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:39,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:39,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:40,832 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:41,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:41,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:42,662 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:43,225 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:43,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:44,833 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:45,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:45,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:47,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:47,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:47,714 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:48,835 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:48,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:19:48,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:19:48,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:19:49,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:49,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:51,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:51,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:52,836 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:53,406 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:53,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:53,696 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:55,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:55,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:56,837 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:19:57,875 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:19:57,907 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:57,918 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:58,727 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:19:59,946 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:19:59,958 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:00,838 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:01,981 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:01,994 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:03,818 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:03,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:20:03,923 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:20:03,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:20:04,020 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:04,035 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:04,067 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:20:04,839 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:06,059 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:06,070 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:08,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:08,105 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:08,830 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:08,840 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:10,131 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:10,144 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:12,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:12,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:12,841 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:13,831 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:14,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:14,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:16,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:16,278 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:16,842 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:18,317 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:18,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:18,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:20:18,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:20:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:20:19,095 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:20,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:20,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:20,843 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:22,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:22,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:22,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:20:22,901 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:20:22,901 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:20:22,903 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:20:23,844 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:20:24,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:24,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:24,844 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:24,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:26,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:26,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:28,846 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:28,928 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:20:28,954 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:28,965 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:30,857 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:30,987 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:30,999 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:32,847 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:33,030 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:33,042 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:33,923 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:20:33,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:20:33,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:20:34,178 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:20:35,065 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:35,077 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:35,859 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:36,848 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:37,102 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:37,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:39,144 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:39,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:40,849 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:41,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:41,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:41,854 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:43,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:43,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:44,850 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:45,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:45,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:46,885 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:47,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:47,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:48,851 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:48,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:20:48,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:20:48,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:20:49,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:49,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:51,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:51,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:51,918 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:52,852 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:53,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:53,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:55,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:55,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:56,853 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:20:56,947 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:20:57,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:57,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:59,920 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:20:59,948 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:20:59,959 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:00,854 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:01,980 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:01,984 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:01,995 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:03,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:21:03,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:21:03,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:21:04,023 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:04,039 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:04,122 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:21:04,855 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:06,062 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:06,074 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:07,028 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:08,101 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:08,112 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:08,857 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:10,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:10,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:12,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:12,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:12,858 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:13,026 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:14,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:14,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:16,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:16,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:16,859 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:18,027 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:18,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:18,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:18,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:21:18,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:21:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:21:20,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:20,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:20,860 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:22,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:22,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:22,860 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:23,838 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:24,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:24,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:26,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:26,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:26,861 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:28,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:28,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:28,914 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:30,862 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:30,948 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:21:30,976 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:30,988 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:33,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:33,026 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:33,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:21:33,924 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:21:33,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:21:34,084 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:21:34,085 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:34,864 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:35,051 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:35,070 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:37,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:37,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:38,865 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:39,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:39,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:39,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:41,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:41,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:42,866 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:43,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:43,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:45,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:45,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:45,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:46,867 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:47,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:47,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:48,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:21:48,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:21:48,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:21:49,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:49,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:50,868 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:51,242 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:51,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:51,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:53,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:53,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:54,869 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:55,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:55,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:56,971 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:21:57,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:57,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:58,870 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:21:59,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:21:59,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:01,051 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:22:01,053 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:22:01,053 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:22:01,054 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:22:01,871 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:22:01,986 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:22:02,029 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:02,041 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:02,054 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:02,871 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:03,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:22:03,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:22:03,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:22:04,073 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:04,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:04,110 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:22:06,122 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:06,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:06,872 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:08,023 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:08,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:08,169 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:10,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:10,206 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:10,873 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:12,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:12,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:13,050 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:14,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:14,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:14,875 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:16,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:16,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:18,064 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:18,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:18,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:18,876 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:18,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:22:18,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:22:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:22:20,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:20,405 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:22,434 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:22,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:22,877 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:24,060 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:24,474 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:24,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:26,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:26,522 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:26,878 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:28,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:28,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:29,090 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:30,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:30,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:30,879 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:33,042 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:22:33,064 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:33,076 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:33,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:22:33,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:22:33,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:22:34,149 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:22:34,150 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:34,880 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:35,103 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:35,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:37,139 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:37,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:38,881 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:39,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:39,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:40,149 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:41,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:41,239 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:42,883 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:43,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:43,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:45,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:45,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:46,125 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:46,884 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:47,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:47,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:48,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:22:48,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:22:48,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:22:49,390 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:49,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:50,885 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:51,153 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:51,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:51,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:53,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:53,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:54,886 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:55,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:55,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:56,192 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:22:57,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:57,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:58,887 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:22:59,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:22:59,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:01,214 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:01,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:01,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:02,888 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:03,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:23:03,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:23:03,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:23:04,063 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:23:04,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:04,100 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:04,151 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:23:06,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:06,151 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:06,890 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:07,196 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:08,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:08,188 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:10,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:10,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:10,891 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:12,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:12,269 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:13,194 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:14,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:14,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:14,892 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:16,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:16,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:18,225 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:18,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:18,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:18,893 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:18,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:23:18,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:23:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:23:20,409 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:20,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:22,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:22,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:22,894 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:24,223 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:24,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:24,512 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:26,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:26,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:26,895 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:28,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:28,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:30,211 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:30,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:30,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:30,897 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:32,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:32,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:33,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:23:33,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:23:33,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:23:34,087 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:23:34,898 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:35,122 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:23:35,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:35,207 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:36,088 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:37,250 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:37,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:38,899 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:39,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:39,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:41,243 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:41,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:41,650 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:23:41,651 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:23:41,652 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:23:41,652 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:23:41,695 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:41,901 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:23:42,901 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:43,747 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:43,766 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:45,809 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:45,824 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:46,614 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:46,902 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:47,853 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:48,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:48,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:23:48,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:23:48,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:23:50,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:50,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:50,904 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:52,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:52,606 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:52,829 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:54,885 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:54,905 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:55,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:57,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:57,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:57,641 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:23:58,906 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:23:59,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:23:59,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:01,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:01,716 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:02,907 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:03,629 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:03,745 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:03,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:03,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:24:03,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:24:03,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:24:04,154 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:24:05,790 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:05,802 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:06,909 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:08,220 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:24:08,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:08,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:08,637 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:10,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:10,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:10,910 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:12,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:12,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:14,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:14,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:14,606 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:14,911 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:16,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:16,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:18,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:18,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:18,913 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:18,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:24:18,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:24:18,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:24:20,084 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:20,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:20,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:22,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:22,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:22,914 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:24,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:24,594 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:25,551 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:26,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:26,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:26,915 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:28,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:28,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:30,700 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:30,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:30,917 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:31,540 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:32,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:32,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:33,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:24:33,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:24:33,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:24:34,119 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:24:34,775 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:34,786 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:34,918 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:36,810 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:36,828 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:37,120 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:38,919 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:39,213 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:24:39,236 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:39,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:41,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:41,289 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:42,552 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:42,920 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:43,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:43,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:45,362 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:45,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:46,921 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:47,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:47,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:48,527 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:48,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:24:48,925 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:24:48,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:24:49,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:49,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:50,923 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:51,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:51,491 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:53,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:53,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:54,488 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:54,924 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:55,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:55,568 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:57,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:57,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:58,925 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:24:59,488 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:24:59,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:24:59,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:01,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:01,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:02,926 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:03,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:03,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:03,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:25:03,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:25:03,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:25:04,081 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:25:05,083 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:05,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:05,769 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:06,929 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:07,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:07,811 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:10,194 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:25:10,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:10,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:10,435 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:10,930 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:12,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:12,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:14,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:14,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:14,931 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:16,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:16,379 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:16,407 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:18,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:18,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:18,925 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:25:18,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:25:18,932 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:18,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:25:20,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:20,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:21,414 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:25:21,415 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:25:21,416 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:25:21,416 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:21,416 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:25:21,933 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:25:22,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:22,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:22,934 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:24,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:24,519 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:26,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:26,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:26,935 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:27,362 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:28,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:28,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:30,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:30,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:30,936 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:32,647 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:32,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:33,340 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:33,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:25:33,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:25:33,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:25:34,075 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:25:34,698 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:34,709 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:34,938 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:36,731 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:36,743 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:38,486 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:38,766 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:38,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:38,939 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:41,167 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:25:41,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:41,202 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:42,940 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:43,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:43,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:44,465 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:45,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:45,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:46,941 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:47,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:47,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:48,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:25:48,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:25:48,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:25:49,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:49,363 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:50,438 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:50,942 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:51,392 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:51,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:53,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:53,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:54,943 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:55,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:55,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:56,436 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:25:57,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:57,527 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:58,945 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:25:59,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:25:59,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:01,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:01,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:02,395 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:02,946 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:03,636 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:03,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:03,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:26:03,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:26:03,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:26:04,122 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:26:05,674 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:05,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:06,947 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:07,710 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:07,726 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:08,389 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:09,757 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:09,768 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:10,948 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:12,150 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:26:12,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:12,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:14,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:14,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:14,364 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:14,949 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:16,262 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:16,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:18,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:18,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:18,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:26:18,926 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:26:18,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:26:18,951 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:20,155 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:20,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:20,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:22,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:22,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:22,952 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:24,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:24,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:25,283 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:26,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:26,473 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:26,953 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:28,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:28,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:30,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:30,579 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:30,953 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:31,240 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:32,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:32,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:33,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:26:33,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:26:33,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:26:34,129 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:26:34,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:34,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:34,954 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:36,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:36,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:37,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:38,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:38,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:38,955 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:40,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:40,772 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:42,957 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:43,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:43,153 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:26:43,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:43,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:45,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:45,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:46,958 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:47,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:47,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:48,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:26:48,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:26:48,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:26:49,204 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:49,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:49,323 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:50,959 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:51,351 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:51,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:53,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:53,405 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:54,960 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:55,020 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:26:55,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:55,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:57,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:57,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:58,961 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:26:59,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:26:59,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:00,948 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:00,997 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:27:00,999 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:27:00,999 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:27:00,999 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:27:01,551 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:01,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:01,962 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:27:02,962 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:03,589 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:03,600 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:03,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:27:03,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:27:03,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:27:04,125 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:27:05,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:05,640 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:06,921 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:06,963 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:07,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:07,683 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:09,710 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:09,722 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:10,964 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:11,748 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:11,760 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:12,868 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:14,137 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:27:14,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:14,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:14,965 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:16,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:16,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:18,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:18,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:18,859 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:18,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:27:18,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:27:18,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:27:18,966 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:20,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:20,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:22,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:22,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:22,967 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:24,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:24,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:24,818 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:26,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:26,435 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:26,968 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:28,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:28,478 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:30,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:30,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:30,812 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:30,969 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:32,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:32,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:33,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:27:33,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:27:33,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:27:34,086 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:27:34,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:34,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:34,970 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:36,087 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:36,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:36,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:38,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:38,676 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:38,971 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:40,700 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:40,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:41,663 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:42,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:42,758 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:42,972 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:45,138 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:27:45,164 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:45,176 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:46,973 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:47,199 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:47,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:47,641 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:48,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:27:48,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:27:48,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:27:49,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:49,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:50,974 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:51,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:51,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:53,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:53,317 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:53,602 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:27:54,975 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:55,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:55,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:57,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:57,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:58,976 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:27:59,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:59,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:27:59,546 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:01,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:01,479 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:02,977 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:03,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:03,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:03,926 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:28:03,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:28:03,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:28:04,088 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:28:05,461 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:05,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:05,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:06,978 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:07,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:07,591 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:08,978 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:09,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:09,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:11,412 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:11,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:11,673 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:12,979 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:13,699 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:13,714 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:16,095 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:28:16,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:16,142 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:16,981 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:17,375 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:18,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:18,186 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:18,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:28:18,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:28:18,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:28:20,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:20,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:20,982 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:22,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:22,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:23,355 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:24,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:24,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:24,983 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:26,350 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:26,368 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:28,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:28,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:28,984 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:29,315 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:30,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:30,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:32,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:32,487 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:32,985 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:33,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:28:33,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:28:33,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:28:34,169 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:28:34,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:34,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:35,170 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:36,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:36,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:36,986 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:38,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:38,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:40,219 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:40,269 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:28:40,270 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:28:40,271 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:28:40,271 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:28:40,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:40,641 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:40,987 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:40,988 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:28:42,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:42,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:44,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:44,719 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:44,988 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:46,164 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:47,094 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:28:47,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:47,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:48,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:28:48,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:28:48,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:28:48,989 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:49,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:49,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:51,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:51,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:52,071 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:52,990 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:53,236 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:53,248 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:55,273 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:55,285 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:56,991 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:57,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:57,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:57,774 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:28:58,992 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:28:59,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:28:59,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:01,391 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:01,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:02,777 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:03,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:03,442 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:03,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:29:03,927 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:29:03,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:29:03,993 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:04,070 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:29:05,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:05,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:07,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:07,522 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:07,994 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:08,721 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:09,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:09,561 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:11,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:11,602 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:11,995 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:13,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:13,643 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:14,659 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:15,675 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:15,687 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:15,997 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:18,069 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:29:18,101 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:18,114 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:18,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:29:18,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:29:18,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:29:19,998 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:20,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:20,146 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:20,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:22,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:22,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:23,999 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:24,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:24,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:25,534 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:26,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:26,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:28,000 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:28,315 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:28,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:30,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:30,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:31,466 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:32,001 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:32,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:32,408 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:33,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:29:33,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:29:33,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:29:34,113 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:29:34,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:34,450 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:36,002 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:36,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:36,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:37,114 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:38,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:38,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:40,003 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:40,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:40,582 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:42,309 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:42,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:42,622 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:44,004 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:44,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:44,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:46,684 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:46,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:48,005 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:48,253 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:48,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:29:48,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:29:48,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:29:49,075 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:29:49,101 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:49,116 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:51,142 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:51,153 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:52,006 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:53,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:53,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:54,155 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:29:55,213 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:55,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:56,007 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:29:57,252 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:57,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:59,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:29:59,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:00,008 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:00,096 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:01,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:01,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:03,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:03,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:03,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:30:03,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:30:03,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:30:04,009 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:04,110 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:30:05,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:05,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:06,019 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:07,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:07,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:08,010 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:09,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:09,502 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:11,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:11,540 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:12,000 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:12,011 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:13,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:13,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:15,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:15,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:16,012 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:16,969 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:30:16,971 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:30:16,971 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:30:16,971 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:30:17,013 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:30:17,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:17,657 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:17,972 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:18,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:30:18,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:30:18,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:30:19,013 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:20,013 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:20,031 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:30:20,061 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:20,075 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:22,113 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:22,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:23,857 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:24,014 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:24,156 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:24,168 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:26,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:26,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:28,015 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:28,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:28,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:29,778 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:30,280 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:30,291 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:32,016 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:32,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:32,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:33,927 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:30:33,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:30:33,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:30:34,130 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:30:34,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:34,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:35,131 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:36,017 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:36,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:36,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:38,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:38,467 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:40,019 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:40,494 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:40,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:40,569 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:42,019 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:42,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:42,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:44,582 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:44,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:46,020 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:46,531 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:46,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:46,633 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:48,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:48,669 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:48,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:30:48,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:30:48,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:30:50,021 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:51,044 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:30:51,066 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:51,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:52,161 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:53,101 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:53,113 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:54,022 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:55,136 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:55,154 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:57,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:57,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:57,352 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:30:58,023 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:30:59,220 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:30:59,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:01,258 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:01,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:02,024 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:03,293 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:03,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:03,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:03,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:31:03,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:31:03,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:31:04,157 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:31:05,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:05,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:06,026 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:07,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:07,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:09,224 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:09,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:09,422 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:10,027 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:11,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:11,473 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:13,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:13,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:14,028 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:15,153 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:15,542 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:15,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:17,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:17,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:18,030 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:18,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:31:18,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:31:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:31:19,625 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:19,640 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:21,036 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:22,010 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:31:22,031 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:22,037 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:22,049 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:24,076 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:24,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:26,032 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:26,117 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:26,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:26,975 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:28,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:28,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:30,033 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:30,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:30,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:32,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:32,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:32,872 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:33,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:31:33,928 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:31:33,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:31:34,034 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:34,151 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:31:34,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:34,296 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:36,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:36,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:38,035 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:38,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:38,385 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:38,789 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:40,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:40,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:42,036 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:42,455 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:42,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:44,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:44,517 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:44,693 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:46,037 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:46,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:46,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:48,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:48,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:48,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:31:48,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:31:48,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:31:50,039 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:50,115 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:50,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:50,634 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:53,008 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:31:53,037 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:53,051 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:54,040 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:55,085 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:55,105 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:55,538 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:31:55,588 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:31:55,589 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:31:55,589 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:31:55,591 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:31:56,040 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:31:57,040 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:57,132 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:57,145 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:58,041 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:31:59,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:31:59,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:01,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:01,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:01,476 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:02,042 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:03,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:03,267 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:03,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:32:03,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:32:03,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:32:04,095 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:32:05,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:05,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:06,043 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:07,096 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:07,329 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:07,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:09,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:09,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:10,044 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:11,397 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:11,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:12,283 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:13,434 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:13,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:14,045 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:15,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:15,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:17,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:17,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:18,046 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:18,212 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:18,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:32:18,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:32:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:32:19,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:19,559 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:21,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:21,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:22,047 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:23,989 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:32:24,019 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:24,034 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:24,048 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:24,087 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:26,064 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:26,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:28,049 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:28,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:28,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:30,047 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:30,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:30,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:32,050 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:32,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:32,204 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:33,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:32:33,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:32:33,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:32:34,061 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:32:34,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:34,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:35,907 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:36,051 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:36,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:36,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:38,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:38,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:40,052 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:40,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:40,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:41,844 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:42,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:42,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:44,053 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:44,439 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:44,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:46,487 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:46,501 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:47,735 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:48,054 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:48,529 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:48,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:48,928 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:32:48,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:32:48,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:32:50,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:50,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:52,055 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:52,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:52,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:53,666 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:32:55,019 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:32:55,044 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:55,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:56,056 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:32:57,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:57,106 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:59,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:59,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:32:59,540 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:00,057 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:01,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:01,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:03,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:03,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:03,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:33:03,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:33:03,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:33:04,058 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:04,106 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:33:05,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:05,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:05,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:07,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:07,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:08,059 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:09,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:09,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:10,345 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:11,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:11,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:12,060 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:13,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:13,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:15,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:15,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:16,061 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:16,261 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:17,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:17,521 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:33:18,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:33:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:33:19,549 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:19,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:20,063 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:21,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:21,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:22,147 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:23,632 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:23,644 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:24,064 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:26,016 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:33:26,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:26,067 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:28,065 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:28,077 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:28,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:28,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:30,135 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:30,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:32,066 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:32,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:32,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:33:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:33,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:33:33,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:33:33,978 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:33:34,071 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:33:34,071 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:33:34,071 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:33:34,072 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:33:34,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:34,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:35,067 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:35,067 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:33:36,067 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:36,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:36,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:38,299 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:38,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:39,899 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:40,068 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:40,340 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:40,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:42,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:42,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:44,069 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:44,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:44,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:45,794 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:46,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:46,482 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:48,070 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:48,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:48,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:48,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:33:48,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:33:48,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:33:50,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:50,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:51,745 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:52,071 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:52,592 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:52,604 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:54,630 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:54,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:56,072 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:33:57,020 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:33:57,046 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:57,066 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:57,590 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:33:59,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:33:59,111 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:00,074 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:01,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:01,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:03,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:03,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:03,535 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:03,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:34:03,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:34:03,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:34:04,074 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:34:04,075 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:05,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:05,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:07,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:07,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:08,076 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:09,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:09,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:09,386 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:10,077 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:11,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:11,366 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:13,399 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:13,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:14,078 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:15,343 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:15,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:15,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:17,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:17,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:18,079 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:34:18,929 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:34:18,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:34:19,541 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:19,563 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:21,146 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:21,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:21,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:22,080 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:23,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:23,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:25,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:25,686 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:26,081 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:27,118 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:28,053 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:34:28,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:28,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:30,082 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:30,135 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:30,147 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:32,176 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:32,188 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:33,000 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:33,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:34:33,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:34:33,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:34:34,084 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:34,107 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:34:34,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:34,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:36,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:36,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:38,085 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:38,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:38,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:38,960 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:40,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:40,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:42,086 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:42,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:42,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:44,432 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:44,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:44,846 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:46,087 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:46,472 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:46,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:48,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:48,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:48,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:34:48,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:34:48,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:34:50,083 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:50,088 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:50,554 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:50,566 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:52,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:52,610 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:54,089 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:54,639 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:54,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:55,690 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:34:56,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:56,697 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:58,090 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:34:59,067 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:34:59,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:34:59,109 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:01,138 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:01,152 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:01,593 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:02,091 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:03,181 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:03,192 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:03,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:35:03,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:35:03,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:35:04,094 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:35:05,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:05,230 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:06,092 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:07,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:07,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:07,514 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:09,297 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:09,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:10,093 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:11,340 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:11,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:12,416 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:35:12,417 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:35:12,418 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:35:12,418 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:35:13,094 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:13,095 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:35:13,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:13,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:13,419 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:14,094 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:15,424 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:15,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:17,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:17,488 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:18,096 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:35:18,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:35:18,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:35:19,116 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:19,514 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:19,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:21,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:21,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:22,097 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:23,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:23,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:24,183 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:25,646 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:25,659 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:26,097 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:27,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:27,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:30,069 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:35:30,098 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:30,099 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:30,110 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:30,130 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:32,139 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:32,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:33,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:35:33,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:35:33,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:35:34,100 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:34,128 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:35:34,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:34,191 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:36,021 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:36,221 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:36,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:38,101 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:38,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:38,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:40,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:40,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:41,952 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:42,102 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:42,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:42,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:44,390 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:44,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:46,103 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:46,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:46,436 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:46,977 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:48,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:48,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:48,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:35:48,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:35:48,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:35:50,104 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:50,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:50,526 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:52,557 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:52,571 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:52,787 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:35:54,105 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:54,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:54,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:56,637 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:56,650 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:58,106 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:35:58,679 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:58,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:35:58,740 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:01,059 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:36:01,099 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:01,113 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:02,107 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:03,146 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:03,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:03,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:36:03,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:36:03,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:36:04,090 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:36:04,091 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:05,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:05,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:06,109 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:07,224 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:07,236 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:08,109 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:09,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:09,278 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:09,547 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:11,306 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:11,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:12,110 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:13,355 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:13,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:15,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:15,402 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:15,417 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:16,111 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:17,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:17,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:18,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:36:18,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:36:18,938 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:36:19,486 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:19,503 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:20,112 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:21,353 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:21,531 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:21,547 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:23,578 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:23,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:24,113 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:25,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:25,635 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:27,208 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:27,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:27,672 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:28,114 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:29,698 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:29,711 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:30,115 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:32,074 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:36:32,102 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:32,113 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:32,924 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:33,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:36:33,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:36:33,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:36:34,067 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:36:34,116 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:34,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:34,155 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:36,187 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:36,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:38,117 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:38,228 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:38,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:38,826 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:40,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:40,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:42,118 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:42,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:42,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:44,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:44,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:44,674 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:46,120 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:46,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:46,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:48,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:48,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:48,656 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:36:48,658 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:36:48,658 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:36:48,659 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:36:48,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:36:48,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:36:48,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:36:49,121 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:36:50,086 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:50,121 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:50,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:50,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:52,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:52,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:54,122 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:54,553 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:54,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:55,521 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:36:56,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:56,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:58,123 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:36:58,642 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:36:58,654 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:00,681 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:00,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:01,318 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:02,124 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:03,065 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:37:03,093 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:03,105 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:03,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:37:03,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:37:03,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:37:04,087 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:37:05,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:05,147 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:06,125 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:07,175 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:07,188 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:07,221 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:09,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:09,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:10,126 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:11,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:11,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:13,002 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:13,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:13,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:14,127 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:15,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:15,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:17,386 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:17,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:18,128 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:18,929 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:18,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:37:18,930 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:37:18,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:37:19,431 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:19,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:21,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:21,483 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:22,130 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:23,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:23,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:24,766 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:25,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:25,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:26,131 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:27,607 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:27,621 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:29,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:29,664 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:30,132 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:30,682 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:31,688 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:31,699 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:32,132 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:33,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:37:33,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:37:33,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:37:34,078 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:37:34,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:34,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:34,139 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:37:36,133 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:36,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:36,163 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:36,490 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:38,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:38,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:40,134 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:40,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:40,239 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:42,263 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:42,278 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:42,393 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:44,136 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:44,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:44,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:46,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:46,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:48,137 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:48,229 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:48,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:48,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:48,930 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:37:48,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:37:48,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:37:50,412 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:50,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:52,138 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:52,448 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:52,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:54,163 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:37:54,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:54,496 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:56,139 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:37:56,520 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:56,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:58,555 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:58,567 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:37:59,973 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:00,140 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:00,593 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:00,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:02,639 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:02,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:03,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:38:03,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:38:03,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:38:04,072 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:38:04,141 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:05,018 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:38:05,046 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:05,062 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:05,073 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:07,093 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:07,108 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:08,142 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:09,138 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:09,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:10,774 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:11,178 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:11,190 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:12,143 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:13,217 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:13,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:15,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:15,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:16,144 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:16,679 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:17,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:17,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:18,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:38:18,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:38:18,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:38:19,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:19,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:20,146 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:21,387 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:21,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:22,080 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:23,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:23,445 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:24,147 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:25,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:25,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:26,482 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:38:26,484 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:38:26,485 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:38:26,485 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:38:27,147 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:38:27,486 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:27,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:27,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:28,148 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:29,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:29,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:31,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:31,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:32,149 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:33,356 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:33,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:33,693 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:33,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:38:33,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:38:33,939 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:38:34,073 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:38:36,064 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:38:36,095 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:36,106 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:36,150 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:38,130 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:38,143 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:39,193 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:40,151 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:40,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:40,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:42,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:42,216 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:44,152 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:44,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:44,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:45,101 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:46,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:46,291 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:48,153 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:48,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:48,324 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:48,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:38:48,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:38:48,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:38:50,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:50,359 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:50,885 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:52,154 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:52,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:52,394 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:54,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:54,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:56,155 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:56,453 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:56,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:56,781 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:38:58,155 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:38:58,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:38:58,505 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:00,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:00,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:02,157 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:02,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:02,572 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:02,602 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:03,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:39:03,931 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:39:03,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:39:04,595 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:04,608 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:04,755 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:39:06,158 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:06,978 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:39:07,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:07,020 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:08,493 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:09,047 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:09,058 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:10,159 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:11,087 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:11,098 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:13,126 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:13,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:14,160 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:14,320 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:15,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:15,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:17,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:17,227 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:18,161 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:18,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:39:18,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:39:18,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:39:19,256 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:19,268 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:19,346 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:21,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:21,312 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:22,162 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:23,339 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:23,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:25,072 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:25,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:25,405 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:26,163 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:27,434 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:27,446 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:29,474 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:29,485 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:30,164 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:30,981 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:31,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:31,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:33,558 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:33,570 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:33,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:39:33,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:39:33,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:39:34,106 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:39:34,165 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:35,598 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:35,613 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:36,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:37,983 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:39:38,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:38,021 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:38,167 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:40,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:40,067 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:41,678 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:42,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:42,108 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:42,168 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:44,136 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:44,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:46,169 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:46,176 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:46,194 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:47,615 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:48,233 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:48,245 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:48,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:39:48,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:39:48,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:39:50,170 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:50,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:50,283 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:52,305 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:52,318 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:53,449 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:39:54,171 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:54,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:54,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:56,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:56,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:58,172 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:39:58,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:58,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:39:59,354 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:00,464 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:00,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:02,173 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:02,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:02,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:03,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:40:03,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:40:03,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:40:04,100 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:40:04,202 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:40:04,204 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:40:04,204 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:40:04,205 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:40:04,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:04,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:05,174 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:40:05,205 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:06,174 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:06,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:06,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:08,955 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:40:08,984 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:08,997 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:10,176 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:11,026 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:11,041 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:11,097 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:13,069 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:13,083 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:14,177 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:15,112 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:15,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:16,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:17,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:17,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:18,178 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:18,931 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:40:18,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:40:18,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:40:19,203 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:19,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:20,178 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:21,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:21,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:22,850 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:23,285 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:23,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:24,179 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:25,330 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:25,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:27,369 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:27,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:28,180 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:28,668 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:29,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:29,422 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:31,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:31,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:32,182 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:33,846 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:33,860 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:33,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:40:33,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:40:33,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:40:34,116 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:40:34,117 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:35,883 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:35,895 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:36,183 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:37,924 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:37,935 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:39,424 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:40,184 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:40,305 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:40:40,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:40,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:42,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:42,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:44,185 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:44,413 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:44,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:45,350 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:46,449 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:46,461 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:48,186 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:48,484 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:48,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:48,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:40:48,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:40:48,940 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:40:50,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:50,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:51,119 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:52,188 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:52,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:52,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:54,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:54,611 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:56,189 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:40:56,635 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:56,658 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:57,102 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:40:58,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:40:58,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:00,190 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:00,732 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:00,744 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:02,767 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:02,780 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:02,904 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:03,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:41:03,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:41:03,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:41:04,165 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:41:04,191 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:04,806 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:04,820 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:06,848 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:06,863 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:08,192 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:08,833 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:08,889 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:08,906 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:11,275 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:41:11,302 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:11,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:12,193 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:13,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:13,354 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:14,652 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:15,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:15,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:16,194 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:17,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:17,441 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:18,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:41:18,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:41:18,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:41:19,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:19,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:20,097 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:20,195 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:21,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:21,524 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:23,556 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:23,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:24,196 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:25,490 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:25,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:25,611 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:27,638 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:27,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:28,197 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:29,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:29,697 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:31,338 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:31,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:31,740 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:32,198 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:33,770 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:33,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:33,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:41:33,932 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:41:33,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:41:34,127 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:41:35,822 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:35,835 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:36,199 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:37,265 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:37,869 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:37,880 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:39,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:39,920 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:40,200 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:42,090 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:41:42,091 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:41:42,092 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:41:42,092 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:41:42,201 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:41:42,304 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:41:42,331 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:42,342 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:43,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:44,202 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:44,367 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:44,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:46,402 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:46,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:48,094 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:48,443 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:48,454 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:48,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:41:48,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:41:48,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:41:50,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:50,489 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:52,513 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:52,525 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:53,681 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:41:54,204 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:41:54,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:54,560 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:56,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:56,596 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:58,620 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:58,631 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:41:58,682 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:00,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:00,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:02,691 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:02,702 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:03,682 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:03,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:42:03,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:42:03,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:42:04,109 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:42:04,725 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:04,738 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:06,207 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:42:06,768 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:06,780 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:08,809 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:08,821 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:09,111 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:10,846 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:10,860 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:13,228 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:42:13,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:13,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:14,112 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:15,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:15,313 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:16,210 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:42:17,344 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:17,356 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:18,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:42:18,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:42:18,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:42:19,123 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:19,382 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:19,393 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:21,420 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:21,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:23,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:23,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:24,124 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:25,499 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:25,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:26,212 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:42:27,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:27,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:29,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:29,587 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:29,797 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:31,612 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:31,624 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:33,652 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:33,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:33,932 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:42:33,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:42:33,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:42:34,174 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:42:35,175 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:35,692 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:35,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:36,215 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:42:37,734 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:37,747 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:39,771 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:39,782 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:40,776 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:41,808 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:41,823 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:44,187 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:42:44,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:44,225 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:45,777 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:46,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:46,267 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:48,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:48,308 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:48,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:42:48,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:42:48,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:42:50,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:50,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:51,158 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:52,372 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:52,384 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:54,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:54,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:56,220 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:42:56,457 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:56,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:56,469 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:42:58,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:42:58,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:00,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:00,545 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:01,470 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:02,569 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:02,581 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:03,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:43:03,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:43:03,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:43:04,094 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:43:04,606 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:04,618 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:06,223 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:43:06,648 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:06,660 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:06,909 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:08,685 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:08,696 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:10,722 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:10,736 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:11,910 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:12,761 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:12,772 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:15,141 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:43:15,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:15,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:17,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:17,222 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:17,515 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:18,226 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:43:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:43:18,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:43:18,941 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:43:19,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:19,265 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:21,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:21,304 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:23,149 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:23,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:23,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:25,375 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:25,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:27,415 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:27,427 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:28,165 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:28,228 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:43:29,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:29,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:31,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:31,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:33,166 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:33,533 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:33,544 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:33,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:43:33,933 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:43:33,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:43:34,181 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:43:35,572 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:35,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:37,609 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:37,626 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:38,231 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:43:38,880 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:39,651 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:39,670 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:41,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:41,705 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:43,731 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:43,742 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:43,880 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:46,109 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:43:46,140 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:46,154 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:48,185 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:48,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:48,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:43:48,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:43:48,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:43:49,148 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:50,229 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:50,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:51,234 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:43:52,266 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:52,277 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:54,149 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:43:54,303 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:54,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:56,346 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:56,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:58,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:58,395 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:43:59,381 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:00,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:00,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:01,237 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:02,458 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:02,470 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:03,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:44:03,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:44:03,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:44:04,130 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:44:04,498 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:04,510 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:05,131 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:06,536 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:06,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:08,575 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:08,586 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:10,587 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:10,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:10,629 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:11,239 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:12,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:12,665 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:14,690 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:14,703 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:15,588 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:17,071 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:44:17,096 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:17,110 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:18,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:44:18,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:44:18,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:44:19,138 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:19,150 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:21,119 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:21,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:21,189 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:23,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:23,231 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:25,243 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:25,263 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:25,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:26,225 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:27,300 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:27,311 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:29,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:29,348 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:31,225 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:31,371 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:31,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:33,407 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:33,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:33,933 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:44:33,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:44:33,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:44:34,091 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:44:35,246 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:35,447 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:35,459 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:37,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:37,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:37,493 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:39,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:39,528 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:41,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:41,562 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:42,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:43,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:43,597 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:45,248 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:45,619 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:45,630 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:47,280 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:47,997 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:44:48,027 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:48,039 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:48,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:44:48,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:44:48,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:44:50,071 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:50,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:52,117 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:52,129 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:53,065 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:54,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:54,173 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:55,251 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:44:56,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:56,214 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:58,066 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:44:58,243 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:44:58,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:00,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:00,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:02,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:02,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:03,066 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:03,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:45:03,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:45:03,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:45:04,093 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:45:04,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:04,381 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:05,253 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:45:06,411 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:06,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:08,094 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:08,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:08,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:10,492 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:10,507 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:12,535 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:12,548 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:13,095 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:14,577 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:14,590 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:16,617 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:16,635 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:17,257 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:45:18,458 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:18,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:45:18,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:45:18,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:45:19,015 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:45:19,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:19,056 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:21,086 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:21,097 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:23,125 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:23,141 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:24,131 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:25,167 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:25,179 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:27,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:27,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:27,260 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:45:29,242 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:29,253 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:29,538 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:31,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:31,294 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:33,316 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:33,328 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:33,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:45:33,934 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:45:33,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:45:34,094 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:45:35,095 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:35,352 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:35,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:37,262 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:45:37,388 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:37,400 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:39,423 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:39,440 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:40,998 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:41,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:41,475 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:43,497 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:43,508 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:45,532 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:45,543 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:45,998 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:47,573 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:47,584 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:48,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:45:48,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:45:48,942 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:45:49,265 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:45:49,961 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:45:49,986 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:50,001 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:51,106 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:52,029 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:52,046 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:54,071 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:54,084 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:56,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:45:56,114 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:56,128 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:58,158 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:58,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:45:59,268 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:00,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:00,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:01,983 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:02,240 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:02,252 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:03,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:46:03,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:46:03,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:46:04,152 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:46:04,280 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:04,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:06,321 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:06,335 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:07,153 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:08,364 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:08,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:09,270 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:10,403 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:10,421 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:12,451 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:12,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:12,621 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:14,490 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:14,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:16,534 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:16,546 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:17,621 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:18,574 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:18,585 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:18,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:46:18,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:46:18,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:46:20,962 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:46:20,991 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:21,005 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:21,273 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:23,032 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:23,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:23,574 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:25,071 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:25,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:27,121 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:27,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:28,575 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:29,160 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:29,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:31,198 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:31,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:31,276 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:33,241 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:33,254 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:33,934 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:46:33,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:46:33,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:46:34,161 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:46:34,164 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:35,278 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:35,295 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:37,319 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:37,336 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:39,163 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:39,357 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:39,370 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:41,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:41,410 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:43,279 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:43,433 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:43,444 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:44,989 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:45,469 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:45,481 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:47,504 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:47,516 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:48,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:46:48,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:46:48,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:46:49,539 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:49,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:50,071 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:51,936 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:46:51,958 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:51,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:53,281 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:46:53,997 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:54,011 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:55,630 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:46:56,040 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:56,051 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:58,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:46:58,090 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:00,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:00,126 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:00,630 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:02,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:02,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:03,284 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:03,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:47:03,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:47:03,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:47:04,099 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:47:04,200 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:04,215 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:06,100 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:06,243 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:06,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:08,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:08,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:10,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:10,337 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:11,101 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:12,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:12,377 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:14,414 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:14,425 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:15,287 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:16,456 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:16,468 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:16,548 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:18,498 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:18,509 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:18,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:47:18,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:47:18,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:47:20,537 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:20,550 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:22,144 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:22,926 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:47:22,955 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:22,967 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:24,997 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:25,009 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:25,290 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:27,041 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:27,052 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:28,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:29,079 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:29,092 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:31,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:31,135 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:33,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:33,165 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:33,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:33,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:47:33,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:47:33,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:47:34,125 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:47:35,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:35,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:37,246 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:37,257 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:37,293 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:38,695 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:39,281 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:39,292 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:41,314 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:41,327 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:43,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:43,365 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:43,696 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:45,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:45,401 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:47,296 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:47,426 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:47,437 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:48,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:47:48,935 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:47:48,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:47:49,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:49,465 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:49,477 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:51,506 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:51,518 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:53,891 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:47:53,917 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:53,933 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:54,093 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:47:55,959 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:55,971 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:57,994 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:58,006 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:47:59,299 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:47:59,547 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:00,032 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:00,046 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:02,073 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:02,090 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:03,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:48:03,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:48:03,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:48:04,120 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:04,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:04,139 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:48:05,140 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:06,163 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:06,177 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:08,205 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:08,219 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:09,301 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:48:10,248 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:10,262 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:10,565 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:12,290 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:12,307 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:14,333 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:14,347 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:15,566 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:16,376 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:16,389 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:18,416 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:18,429 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:18,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:48:18,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:48:18,943 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:48:19,304 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:48:20,460 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:20,471 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:21,142 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:22,503 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:22,515 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:24,882 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:48:24,912 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:24,924 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:26,143 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:26,950 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:26,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:28,992 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:29,004 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:31,032 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:31,047 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:31,307 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:48:31,776 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:33,074 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:33,085 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:33,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:48:33,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:48:33,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:48:34,140 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:48:35,113 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:35,124 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:37,141 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:37,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:37,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:39,198 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:39,210 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:41,237 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:41,249 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:41,310 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:48:42,412 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:43,272 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:43,284 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:45,309 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:45,325 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:47,349 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:47,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:47,413 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:48,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:48:48,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:48:48,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:48:49,383 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:49,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:51,312 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:48:51,418 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:51,430 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:52,495 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:53,452 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:53,463 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:55,832 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:48:55,862 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:55,875 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:57,495 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:48:57,900 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:57,913 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:59,938 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:48:59,949 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:01,315 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:01,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:01,984 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:02,702 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:03,935 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:49:03,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:49:03,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:49:04,005 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:04,018 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:04,107 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:49:06,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:06,055 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:08,082 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:08,094 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:08,108 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:10,120 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:10,133 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:11,318 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:12,161 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:12,172 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:13,267 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:14,197 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:14,212 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:16,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:16,255 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:18,268 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:18,282 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:18,293 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:18,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:49:18,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:49:18,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:49:20,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:20,332 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:22,360 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:22,378 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:23,321 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:24,029 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:24,404 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:24,420 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:26,788 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:49:26,815 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:26,828 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:28,856 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:28,867 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:29,030 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:30,897 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:30,908 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:32,938 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:32,960 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:33,324 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:33,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:49:33,936 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:49:33,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:49:34,101 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:34,106 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:49:34,987 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:35,006 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:37,033 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:37,048 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:39,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:39,089 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:39,108 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:41,123 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:41,134 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:43,162 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:43,174 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:44,317 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:45,198 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:45,218 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:45,327 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:47,244 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:47,261 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:48,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:49:48,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:49:48,944 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:49:49,286 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:49,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:50,126 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:51,320 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:51,334 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:53,358 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:53,374 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:55,329 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:49:55,396 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:55,419 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:55,499 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:49:57,789 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:49:57,812 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:57,823 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:59,847 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:49:59,859 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:00,499 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:01,884 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:01,899 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:03,921 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:03,932 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:03,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:50:03,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:50:03,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:50:04,149 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:50:05,332 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:50:05,955 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:05,966 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:06,150 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:07,989 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:08,001 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:10,025 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:10,037 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:11,151 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:12,059 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:12,073 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:14,098 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:14,110 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:15,335 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:50:16,138 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:16,149 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:16,307 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:18,183 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:18,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:50:18,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:50:18,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:50:20,220 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:20,234 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:22,069 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:22,259 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:22,270 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:24,298 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:24,310 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:25,337 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:50:26,341 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:26,353 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:27,726 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:28,718 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:50:28,746 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:28,768 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:30,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:30,809 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:32,726 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:32,838 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:32,849 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:33,936 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:50:33,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:50:33,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:50:34,109 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:50:34,881 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:34,893 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:35,340 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:50:36,926 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:36,937 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:38,111 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:38,962 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:38,973 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:41,003 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:41,014 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:43,044 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:43,056 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:43,111 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:45,086 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:45,098 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:47,130 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:47,141 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:48,112 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:48,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:50:48,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:50:48,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:50:49,171 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:49,182 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:51,208 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:51,220 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:53,154 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:53,251 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:53,263 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:55,288 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:55,301 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:55,346 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:50:57,326 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:57,343 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:58,545 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:50:59,716 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:50:59,739 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:50:59,750 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:01,776 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:01,789 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:03,546 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:03,818 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:03,830 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:03,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:51:03,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:51:03,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:51:04,160 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:51:05,852 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:05,865 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:07,349 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:51:07,892 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:07,906 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:08,995 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:09,927 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:09,939 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:11,963 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:11,980 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:13,995 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:14,003 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:14,014 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:16,037 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:16,049 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:18,072 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:18,083 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:51:18,937 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:51:18,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:51:19,278 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:20,107 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:20,119 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:22,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:22,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:24,184 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:24,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:24,279 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:26,223 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:26,235 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:27,354 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:51:28,260 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:28,271 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:29,731 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:30,640 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:51:30,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:30,680 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:32,708 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:32,724 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:33,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:51:33,938 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:51:33,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:51:34,169 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:51:34,753 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:34,769 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:35,170 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:36,797 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:36,808 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:37,356 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:51:38,833 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:38,844 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:40,567 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:40,878 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:40,889 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:42,920 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:42,931 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:44,961 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:44,972 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:45,568 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:47,005 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:47,017 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:48,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:51:48,938 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:51:48,945 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:51:49,050 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:49,062 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:49,359 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:51:51,097 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:51,108 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:51,118 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:53,137 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:53,148 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:55,180 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:55,196 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:56,119 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:51:57,226 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:57,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:59,264 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:59,276 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:51:59,362 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:01,643 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:52:01,671 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:01,682 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:01,689 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:03,706 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:03,718 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:03,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:52:03,938 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:52:03,946 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:52:04,168 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:52:05,741 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:05,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:07,169 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:07,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:07,795 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:09,365 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:09,819 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:09,832 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:11,855 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:11,868 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:12,971 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:13,894 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:13,907 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:15,931 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:15,945 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:17,970 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:17,972 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:17,982 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:18,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:52:18,938 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:52:18,946 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:52:20,010 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:20,021 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:21,368 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:22,043 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:22,054 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:23,106 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:24,078 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:24,092 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:26,115 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:26,131 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:28,107 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:28,157 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:28,169 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:30,195 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:30,207 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:31,371 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:32,583 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:52:32,616 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:32,628 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:33,937 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:52:33,938 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:52:33,946 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:52:34,079 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:34,115 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:52:34,653 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:34,664 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:36,695 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:36,712 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:38,736 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:38,749 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:39,117 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:40,777 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:40,788 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:41,374 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:42,814 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:42,825 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:43,235 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: partial_history +2024-01-22 20:52:43,237 DEBUG SenderThread:123548 [sender.py:send():382] send: history +2024-01-22 20:52:43,237 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: summary_record +2024-01-22 20:52:43,238 INFO SenderThread:123548 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-01-22 20:52:43,375 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/wandb-summary.json +2024-01-22 20:52:44,239 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:44,375 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:44,856 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:44,870 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:45,376 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:46,899 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:46,922 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:48,952 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:48,964 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:49,240 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:51,002 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:51,127 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:53,159 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:53,170 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:54,240 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:52:55,198 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:55,211 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:55,252 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:52:55,252 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:52:55,253 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:52:57,238 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:57,274 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:59,303 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:59,380 INFO Thread-12 :123548 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sanchit/whisper-large-v3-ft-cv16-mn/wandb/run-20240122_192233-r12o64xn/files/output.log +2024-01-22 20:52:59,398 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:52:59,993 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:01,428 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:01,605 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:03,978 ERROR gpu :123548 [interfaces.py:aggregate():161] Failed to serialize metric: division by zero +2024-01-22 20:53:04,008 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:04,022 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:04,271 DEBUG SenderThread:123548 [sender.py:send():382] send: stats +2024-01-22 20:53:05,272 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:06,052 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:06,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:08,373 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:08,511 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:10,273 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:10,338 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:53:10,538 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:10,721 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:10,840 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:53:10,840 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:53:12,752 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:12,889 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:14,919 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:14,931 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:15,997 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:16,961 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:17,063 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:19,093 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:19,201 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:20,998 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:21,232 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:21,438 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:23,466 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:23,564 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:25,588 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:25,599 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:25,615 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: internal_messages +2024-01-22 20:53:25,707 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: stop_status +2024-01-22 20:53:25,708 DEBUG SenderThread:123548 [sender.py:send_request():409] send_request: stop_status +2024-01-22 20:53:26,873 DEBUG HandlerThread:123548 [handler.py:handle_request():146] handle_request: status_report +2024-01-22 20:53:27,623 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:27,667 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:29,694 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported +2024-01-22 20:53:29,796 ERROR gpu :123548 [interfaces.py:monitor():144] Failed to sample metric: Not Supported