Training in progress, epoch 1

adapter_config.json

@@ -0,0 +1,32 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "gate_proj",
+    "up_proj",
+    "o_proj",
+    "down_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a531bff2e9e9e313dfa3129130f71aadb6f57b5679cd77dfd5eab57a03f2280
+size 75013128

output-67b-26k-lora/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

output-67b-26k-lora/.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

output-67b-26k-lora/runs/Feb08_17-59-25_default/events.out.tfevents.1707415219.default.1738.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f625b5d6cd7f38271c6d7aef4793ad4ff8b57cb355a1b0755d8b8e24fa7c7216
+size 4301

output-7b-26k-lora/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

output-7b-26k-lora/.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

output-7b-26k-lora/runs/Feb08_17-51-59_default/events.out.tfevents.1707414733.default.1048.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31ba9262375486ec9bec3476ffab5b0c89311d7fb13223a5984f1d5ff153e147
+size 4262

output-7b-26k-lora/runs/Feb08_18-45-12_default/events.out.tfevents.1707417920.default.608.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64454a82c3341170925444f73e1611ed9c04146aa5d26f1acdbce8ad2b316f8c
+size 4258

output-7b-26k-lora/runs/Feb08_18-53-01_default/events.out.tfevents.1707418390.default.1281.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49d888485c31b1d6871d434d1a8ae9a3bad0bac375993898cc42ee5514329d29
+size 4258

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|EOT|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,146 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "100000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100002": {
+      "content": "ø",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100003": {
+      "content": "ö",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100004": {
+      "content": "ú",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100005": {
+      "content": "ÿ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100006": {
+      "content": "õ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100007": {
+      "content": "÷",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100008": {
+      "content": "û",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100009": {
+      "content": "ý",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100010": {
+      "content": "À",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100011": {
+      "content": "ù",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100012": {
+      "content": "Á",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100013": {
+      "content": "þ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100014": {
+      "content": "ü",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100015": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|EOT|>",
+  "legacy": true,
+  "model_max_length": 960,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e927268e56d86c7a3a121e9db4be32d47966e50d2f759740a43c21abfd18f9a1
+size 4792

wandb/debug-internal.log

@@ -0,0 +1,194 @@
+2024-02-08 18:53:11,296 INFO    StreamThr :1516 [internal.py:wandb_internal():86] W&B internal server running at pid: 1516, started at: 2024-02-08 18:53:11.296140
+2024-02-08 18:53:11,300 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: status
+2024-02-08 18:53:11,301 INFO    WriterThread:1516 [datastore.py:open_for_write():85] open: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/run-5uym8l7w.wandb
+2024-02-08 18:53:11,302 DEBUG   SenderThread:1516 [sender.py:send():382] send: header
+2024-02-08 18:53:11,302 DEBUG   SenderThread:1516 [sender.py:send():382] send: run
+2024-02-08 18:53:11,515 INFO    SenderThread:1516 [dir_watcher.py:__init__():211] watching files in: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files
+2024-02-08 18:53:11,515 INFO    SenderThread:1516 [sender.py:_start_run_threads():1136] run started: 5uym8l7w with start time 1707418391.295585
+2024-02-08 18:53:11,519 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: check_version
+2024-02-08 18:53:11,520 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: check_version
+2024-02-08 18:53:11,603 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: run_start
+2024-02-08 18:53:11,632 DEBUG   HandlerThread:1516 [system_info.py:__init__():32] System info init
+2024-02-08 18:53:11,633 DEBUG   HandlerThread:1516 [system_info.py:__init__():47] System info init done
+2024-02-08 18:53:11,633 INFO    HandlerThread:1516 [system_monitor.py:start():194] Starting system monitor
+2024-02-08 18:53:11,633 INFO    SystemMonitor:1516 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-02-08 18:53:11,633 INFO    HandlerThread:1516 [system_monitor.py:probe():214] Collecting system info
+2024-02-08 18:53:11,634 INFO    SystemMonitor:1516 [interfaces.py:start():190] Started cpu monitoring
+2024-02-08 18:53:11,635 INFO    SystemMonitor:1516 [interfaces.py:start():190] Started disk monitoring
+2024-02-08 18:53:11,636 INFO    SystemMonitor:1516 [interfaces.py:start():190] Started gpu monitoring
+2024-02-08 18:53:11,637 INFO    SystemMonitor:1516 [interfaces.py:start():190] Started memory monitoring
+2024-02-08 18:53:11,637 INFO    SystemMonitor:1516 [interfaces.py:start():190] Started network monitoring
+2024-02-08 18:53:11,692 DEBUG   HandlerThread:1516 [system_info.py:probe():196] Probing system
+2024-02-08 18:53:11,694 DEBUG   HandlerThread:1516 [gitlib.py:_init_repo():56] git repository is invalid
+2024-02-08 18:53:11,694 DEBUG   HandlerThread:1516 [system_info.py:probe():244] Probing system done
+2024-02-08 18:53:11,694 DEBUG   HandlerThread:1516 [system_monitor.py:probe():223] {'os': 'Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35', 'python': '3.10.13', 'heartbeatAt': '2024-02-08T18:53:11.692683', 'startedAt': '2024-02-08T18:53:11.292051', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py', 'codePathLocal': None, 'host': 'default', 'username': 'sagemaker-user', 'executable': '/opt/conda/bin/python3', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 3259.3440989583337, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 3299.457, 'min': 0.0, 'max': 0.0}, {'current': 3299.806, 'min': 0.0, 'max': 0.0}, {'current': 3205.787, 'min': 0.0, 'max': 0.0}, {'current': 3178.002, 'min': 0.0, 'max': 0.0}, {'current': 3200.706, 'min': 0.0, 'max': 0.0}, {'current': 3196.209, 'min': 0.0, 'max': 0.0}, {'current': 3188.711, 'min': 0.0, 'max': 0.0}, {'current': 3189.097, 'min': 0.0, 'max': 0.0}, {'current': 3206.992, 'min': 0.0, 'max': 0.0}, {'current': 3200.764, 'min': 0.0, 'max': 0.0}, {'current': 2810.939, 'min': 0.0, 'max': 0.0}, {'current': 2847.942, 'min': 0.0, 'max': 0.0}, {'current': 2923.114, 'min': 0.0, 'max': 0.0}, {'current': 3028.23, 'min': 0.0, 'max': 0.0}, {'current': 3012.472, 'min': 0.0, 'max': 0.0}, {'current': 3044.914, 'min': 0.0, 'max': 0.0}, {'current': 2953.37, 'min': 0.0, 'max': 0.0}, {'current': 2957.586, 'min': 0.0, 'max': 0.0}, {'current': 2984.294, 'min': 0.0, 'max': 0.0}, {'current': 2961.352, 'min': 0.0, 'max': 0.0}, {'current': 2901.559, 'min': 0.0, 'max': 0.0}, {'current': 2801.726, 'min': 0.0, 'max': 0.0}, {'current': 2985.17, 'min': 0.0, 'max': 0.0}, {'current': 2963.11, 'min': 0.0, 'max': 0.0}, {'current': 2912.001, 'min': 0.0, 'max': 0.0}, {'current': 2965.712, 'min': 0.0, 'max': 0.0}, {'current': 2966.821, 'min': 0.0, 'max': 0.0}, {'current': 2871.172, 'min': 0.0, 'max': 0.0}, {'current': 2974.758, 'min': 0.0, 'max': 0.0}, {'current': 2989.099, 'min': 0.0, 'max': 0.0}, {'current': 2948.999, 'min': 0.0, 'max': 0.0}, {'current': 2895.266, 'min': 0.0, 'max': 0.0}, {'current': 3299.988, 'min': 0.0, 'max': 0.0}, {'current': 2924.435, 'min': 0.0, 'max': 0.0}, {'current': 2919.839, 'min': 0.0, 'max': 0.0}, {'current': 2875.943, 'min': 0.0, 'max': 0.0}, {'current': 3300.697, 'min': 0.0, 'max': 0.0}, {'current': 2805.016, 'min': 0.0, 'max': 0.0}, {'current': 3298.583, 'min': 0.0, 'max': 0.0}, {'current': 3298.604, 'min': 0.0, 'max': 0.0}, {'current': 2673.256, 'min': 0.0, 'max': 0.0}, {'current': 3296.503, 'min': 0.0, 'max': 0.0}, {'current': 3139.11, 'min': 0.0, 'max': 0.0}, {'current': 3137.942, 'min': 0.0, 'max': 0.0}, {'current': 2833.969, 'min': 0.0, 'max': 0.0}, {'current': 3153.277, 'min': 0.0, 'max': 0.0}, {'current': 3178.769, 'min': 0.0, 'max': 0.0}, {'current': 3207.604, 'min': 0.0, 'max': 0.0}, {'current': 2892.532, 'min': 0.0, 'max': 0.0}, {'current': 3299.772, 'min': 0.0, 'max': 0.0}, {'current': 3299.641, 'min': 0.0, 'max': 0.0}, {'current': 3300.096, 'min': 0.0, 'max': 0.0}, {'current': 3298.515, 'min': 0.0, 'max': 0.0}, {'current': 3298.26, 'min': 0.0, 'max': 0.0}, {'current': 3299.084, 'min': 0.0, 'max': 0.0}, {'current': 3298.903, 'min': 0.0, 'max': 0.0}, {'current': 3298.866, 'min': 0.0, 'max': 0.0}, {'current': 3300.276, 'min': 0.0, 'max': 0.0}, {'current': 3298.704, 'min': 0.0, 'max': 0.0}, {'current': 3299.342, 'min': 0.0, 'max': 0.0}, {'current': 3297.795, 'min': 0.0, 'max': 0.0}, {'current': 3297.923, 'min': 0.0, 'max': 0.0}, {'current': 3298.013, 'min': 0.0, 'max': 0.0}, {'current': 3297.43, 'min': 0.0, 'max': 0.0}, {'current': 3299.564, 'min': 0.0, 'max': 0.0}, {'current': 3300.54, 'min': 0.0, 'max': 0.0}, {'current': 2747.263, 'min': 0.0, 'max': 0.0}, {'current': 3299.353, 'min': 0.0, 'max': 0.0}, {'current': 3297.896, 'min': 0.0, 'max': 0.0}, {'current': 2533.725, 'min': 0.0, 'max': 0.0}, {'current': 3299.656, 'min': 0.0, 'max': 0.0}, {'current': 3293.031, 'min': 0.0, 'max': 0.0}, {'current': 3027.834, 'min': 0.0, 'max': 0.0}, {'current': 3024.556, 'min': 0.0, 'max': 0.0}, {'current': 3067.379, 'min': 0.0, 'max': 0.0}, {'current': 3010.826, 'min': 0.0, 'max': 0.0}, {'current': 3101.81, 'min': 0.0, 'max': 0.0}, {'current': 2973.599, 'min': 0.0, 'max': 0.0}, {'current': 3061.27, 'min': 0.0, 'max': 0.0}, {'current': 3291.322, 'min': 0.0, 'max': 0.0}, {'current': 3017.723, 'min': 0.0, 'max': 0.0}, {'current': 2660.496, 'min': 0.0, 'max': 0.0}, {'current': 3004.775, 'min': 0.0, 'max': 0.0}, {'current': 3021.086, 'min': 0.0, 'max': 0.0}, {'current': 3027.592, 'min': 0.0, 'max': 0.0}, {'current': 3059.589, 'min': 0.0, 'max': 0.0}, {'current': 3019.568, 'min': 0.0, 'max': 0.0}, {'current': 3029.623, 'min': 0.0, 'max': 0.0}, {'current': 3080.312, 'min': 0.0, 'max': 0.0}, {'current': 3066.263, 'min': 0.0, 'max': 0.0}, {'current': 2998.37, 'min': 0.0, 'max': 0.0}, {'current': 2949.133, 'min': 0.0, 'max': 0.0}, {'current': 2964.0, 'min': 0.0, 'max': 0.0}, {'current': 3222.788, 'min': 0.0, 'max': 0.0}, {'current': 3299.63, 'min': 0.0, 'max': 0.0}, {'current': 2916.281, 'min': 0.0, 'max': 0.0}, {'current': 2825.282, 'min': 0.0, 'max': 0.0}, {'current': 3038.106, 'min': 0.0, 'max': 0.0}, {'current': 2895.235, 'min': 0.0, 'max': 0.0}, {'current': 3092.874, 'min': 0.0, 'max': 0.0}, {'current': 2924.994, 'min': 0.0, 'max': 0.0}, {'current': 2913.404, 'min': 0.0, 'max': 0.0}, {'current': 2935.638, 'min': 0.0, 'max': 0.0}, {'current': 2583.261, 'min': 0.0, 'max': 0.0}, {'current': 3101.162, 'min': 0.0, 'max': 0.0}, {'current': 3063.704, 'min': 0.0, 'max': 0.0}, {'current': 3093.23, 'min': 0.0, 'max': 0.0}, {'current': 3095.386, 'min': 0.0, 'max': 0.0}, {'current': 2925.773, 'min': 0.0, 'max': 0.0}, {'current': 2920.019, 'min': 0.0, 'max': 0.0}, {'current': 2916.15, 'min': 0.0, 'max': 0.0}, {'current': 2944.025, 'min': 0.0, 'max': 0.0}, {'current': 3259.667, 'min': 0.0, 'max': 0.0}, {'current': 3049.572, 'min': 0.0, 'max': 0.0}, {'current': 3263.675, 'min': 0.0, 'max': 0.0}, {'current': 3074.497, 'min': 0.0, 'max': 0.0}, {'current': 2923.985, 'min': 0.0, 'max': 0.0}, {'current': 2910.425, 'min': 0.0, 'max': 0.0}, {'current': 2812.861, 'min': 0.0, 'max': 0.0}, {'current': 2874.988, 'min': 0.0, 'max': 0.0}, {'current': 3120.953, 'min': 0.0, 'max': 0.0}, {'current': 3124.25, 'min': 0.0, 'max': 0.0}, {'current': 3113.753, 'min': 0.0, 'max': 0.0}, {'current': 3119.282, 'min': 0.0, 'max': 0.0}, {'current': 2982.281, 'min': 0.0, 'max': 0.0}, {'current': 3048.291, 'min': 0.0, 'max': 0.0}, {'current': 2987.986, 'min': 0.0, 'max': 0.0}, {'current': 2733.968, 'min': 0.0, 'max': 0.0}, {'current': 3274.202, 'min': 0.0, 'max': 0.0}, {'current': 3120.154, 'min': 0.0, 'max': 0.0}, {'current': 3122.388, 'min': 0.0, 'max': 0.0}, {'current': 2592.46, 'min': 0.0, 'max': 0.0}, {'current': 3121.448, 'min': 0.0, 'max': 0.0}, {'current': 3085.363, 'min': 0.0, 'max': 0.0}, {'current': 3176.23, 'min': 0.0, 'max': 0.0}, {'current': 3098.413, 'min': 0.0, 'max': 0.0}, {'current': 3131.838, 'min': 0.0, 'max': 0.0}, {'current': 3297.418, 'min': 0.0, 'max': 0.0}, {'current': 3144.573, 'min': 0.0, 'max': 0.0}, {'current': 3142.177, 'min': 0.0, 'max': 0.0}, {'current': 3135.089, 'min': 0.0, 'max': 0.0}, {'current': 3124.315, 'min': 0.0, 'max': 0.0}, {'current': 3206.745, 'min': 0.0, 'max': 0.0}, {'current': 3197.608, 'min': 0.0, 'max': 0.0}, {'current': 3271.659, 'min': 0.0, 'max': 0.0}, {'current': 3055.483, 'min': 0.0, 'max': 0.0}, {'current': 3299.813, 'min': 0.0, 'max': 0.0}, {'current': 3299.316, 'min': 0.0, 'max': 0.0}, {'current': 3298.471, 'min': 0.0, 'max': 0.0}, {'current': 3275.344, 'min': 0.0, 'max': 0.0}, {'current': 3298.318, 'min': 0.0, 'max': 0.0}, {'current': 3272.185, 'min': 0.0, 'max': 0.0}, {'current': 3299.032, 'min': 0.0, 'max': 0.0}, {'current': 3273.055, 'min': 0.0, 'max': 0.0}, {'current': 3277.573, 'min': 0.0, 'max': 0.0}, {'current': 3274.44, 'min': 0.0, 'max': 0.0}, {'current': 3275.925, 'min': 0.0, 'max': 0.0}, {'current': 3279.092, 'min': 0.0, 'max': 0.0}, {'current': 3275.089, 'min': 0.0, 'max': 0.0}, {'current': 3277.671, 'min': 0.0, 'max': 0.0}, {'current': 3299.135, 'min': 0.0, 'max': 0.0}, {'current': 3299.31, 'min': 0.0, 'max': 0.0}, {'current': 3298.038, 'min': 0.0, 'max': 0.0}, {'current': 3218.557, 'min': 0.0, 'max': 0.0}, {'current': 3298.859, 'min': 0.0, 'max': 0.0}, {'current': 3298.545, 'min': 0.0, 'max': 0.0}, {'current': 3027.843, 'min': 0.0, 'max': 0.0}, {'current': 3299.687, 'min': 0.0, 'max': 0.0}, {'current': 3053.229, 'min': 0.0, 'max': 0.0}, {'current': 3299.26, 'min': 0.0, 'max': 0.0}, {'current': 3059.862, 'min': 0.0, 'max': 0.0}, {'current': 3090.937, 'min': 0.0, 'max': 0.0}, {'current': 3094.897, 'min': 0.0, 'max': 0.0}, {'current': 3083.774, 'min': 0.0, 'max': 0.0}, {'current': 3027.722, 'min': 0.0, 'max': 0.0}, {'current': 3303.02, 'min': 0.0, 'max': 0.0}, {'current': 3069.951, 'min': 0.0, 'max': 0.0}, {'current': 3049.694, 'min': 0.0, 'max': 0.0}, {'current': 2814.624, 'min': 0.0, 'max': 0.0}, {'current': 3097.913, 'min': 0.0, 'max': 0.0}, {'current': 2788.423, 'min': 0.0, 'max': 0.0}, {'current': 3299.195, 'min': 0.0, 'max': 0.0}, {'current': 3069.533, 'min': 0.0, 'max': 0.0}, {'current': 3074.679, 'min': 0.0, 'max': 0.0}, {'current': 3066.308, 'min': 0.0, 'max': 0.0}, {'current': 2598.471, 'min': 0.0, 'max': 0.0}, {'current': 3299.109, 'min': 0.0, 'max': 0.0}, {'current': 3299.455, 'min': 0.0, 'max': 0.0}, {'current': 3298.945, 'min': 0.0, 'max': 0.0}, {'current': 3298.926, 'min': 0.0, 'max': 0.0}, {'current': 3299.003, 'min': 0.0, 'max': 0.0}, {'current': 3299.535, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 32.0, 'used': 0.01256561279296875}}, 'gpu': 'NVIDIA A10G', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}], 'memory': {'total': 747.9597625732422}}
+2024-02-08 18:53:11,694 INFO    HandlerThread:1516 [system_monitor.py:probe():224] Finished collecting system info
+2024-02-08 18:53:11,694 INFO    HandlerThread:1516 [system_monitor.py:probe():227] Publishing system info
+2024-02-08 18:53:11,694 DEBUG   HandlerThread:1516 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2024-02-08 18:53:11,695 DEBUG   HandlerThread:1516 [system_info.py:_save_pip():68] Saving pip packages done
+2024-02-08 18:53:11,695 DEBUG   HandlerThread:1516 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2024-02-08 18:53:12,517 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/conda-environment.yaml
+2024-02-08 18:53:12,517 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/requirements.txt
+2024-02-08 18:53:25,997 DEBUG   HandlerThread:1516 [system_info.py:_save_conda():87] Saving conda packages done
+2024-02-08 18:53:25,999 INFO    HandlerThread:1516 [system_monitor.py:probe():229] Finished publishing system info
+2024-02-08 18:53:26,003 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:53:26,003 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:53:26,003 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:53:26,003 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:53:26,003 DEBUG   SenderThread:1516 [sender.py:send():382] send: files
+2024-02-08 18:53:26,004 INFO    SenderThread:1516 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now
+2024-02-08 18:53:26,008 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-08 18:53:26,008 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: stop_status
+2024-02-08 18:53:26,010 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:53:26,169 DEBUG   SenderThread:1516 [sender.py:send():382] send: telemetry
+2024-02-08 18:53:26,169 DEBUG   SenderThread:1516 [sender.py:send():382] send: config
+2024-02-08 18:53:26,170 DEBUG   SenderThread:1516 [sender.py:send():382] send: metric
+2024-02-08 18:53:26,170 DEBUG   SenderThread:1516 [sender.py:send():382] send: telemetry
+2024-02-08 18:53:26,170 DEBUG   SenderThread:1516 [sender.py:send():382] send: metric
+2024-02-08 18:53:26,170 WARNING SenderThread:1516 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen)
+2024-02-08 18:53:26,382 INFO    wandb-upload_0:1516 [upload_job.py:push():131] Uploaded file /tmp/tmpo2kahburwandb/uswugwyb-wandb-metadata.json
+2024-02-08 18:53:26,518 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/conda-environment.yaml
+2024-02-08 18:53:26,518 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/output.log
+2024-02-08 18:53:26,518 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/wandb-metadata.json
+2024-02-08 18:53:26,821 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:53:28,518 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/output.log
+2024-02-08 18:53:28,894 DEBUG   SenderThread:1516 [sender.py:send():382] send: exit
+2024-02-08 18:53:28,894 INFO    SenderThread:1516 [sender.py:send_exit():589] handling exit code: 1
+2024-02-08 18:53:28,894 INFO    SenderThread:1516 [sender.py:send_exit():591] handling runtime: 17
+2024-02-08 18:53:28,894 INFO    SenderThread:1516 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:53:28,894 INFO    SenderThread:1516 [sender.py:send_exit():597] send defer
+2024-02-08 18:53:28,894 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,895 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 0
+2024-02-08 18:53:28,895 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,895 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-02-08 18:53:28,895 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 1
+2024-02-08 18:53:28,895 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,895 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 1
+2024-02-08 18:53:28,895 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,895 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-02-08 18:53:28,895 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 2
+2024-02-08 18:53:28,895 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,895 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 2
+2024-02-08 18:53:28,895 INFO    HandlerThread:1516 [system_monitor.py:finish():203] Stopping system monitor
+2024-02-08 18:53:28,897 DEBUG   SystemMonitor:1516 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-02-08 18:53:28,897 DEBUG   SystemMonitor:1516 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-02-08 18:53:28,897 DEBUG   SystemMonitor:1516 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-02-08 18:53:28,899 INFO    HandlerThread:1516 [interfaces.py:finish():202] Joined cpu monitor
+2024-02-08 18:53:28,899 INFO    HandlerThread:1516 [interfaces.py:finish():202] Joined disk monitor
+2024-02-08 18:53:28,936 INFO    HandlerThread:1516 [interfaces.py:finish():202] Joined gpu monitor
+2024-02-08 18:53:28,936 INFO    HandlerThread:1516 [interfaces.py:finish():202] Joined memory monitor
+2024-02-08 18:53:28,936 INFO    HandlerThread:1516 [interfaces.py:finish():202] Joined network monitor
+2024-02-08 18:53:28,936 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,936 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-02-08 18:53:28,936 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 3
+2024-02-08 18:53:28,937 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,937 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 3
+2024-02-08 18:53:28,937 DEBUG   SenderThread:1516 [sender.py:send():382] send: stats
+2024-02-08 18:53:28,938 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,938 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-02-08 18:53:28,938 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 4
+2024-02-08 18:53:28,938 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,938 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 4
+2024-02-08 18:53:28,938 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,938 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-02-08 18:53:28,938 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 5
+2024-02-08 18:53:28,938 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,938 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 5
+2024-02-08 18:53:28,938 DEBUG   SenderThread:1516 [sender.py:send():382] send: summary
+2024-02-08 18:53:28,939 INFO    SenderThread:1516 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:53:28,940 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,940 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-02-08 18:53:28,940 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 6
+2024-02-08 18:53:28,940 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:28,940 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 6
+2024-02-08 18:53:28,940 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:28,940 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-02-08 18:53:28,944 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:53:29,088 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 7
+2024-02-08 18:53:29,089 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:29,089 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 7
+2024-02-08 18:53:29,089 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:29,089 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-02-08 18:53:29,519 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/config.yaml
+2024-02-08 18:53:29,519 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/wandb-summary.json
+2024-02-08 18:53:29,894 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:53:30,180 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 8
+2024-02-08 18:53:30,180 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:53:30,181 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:30,181 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 8
+2024-02-08 18:53:30,181 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:30,181 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-02-08 18:53:30,181 INFO    SenderThread:1516 [job_builder.py:build():298] Attempting to build job artifact
+2024-02-08 18:53:30,182 INFO    SenderThread:1516 [job_builder.py:_get_source_type():439] no source found
+2024-02-08 18:53:30,182 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 9
+2024-02-08 18:53:30,182 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:30,182 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 9
+2024-02-08 18:53:30,183 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:30,183 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-02-08 18:53:30,183 INFO    SenderThread:1516 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-02-08 18:53:30,519 INFO    Thread-12 :1516 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/output.log
+2024-02-08 18:53:30,519 INFO    SenderThread:1516 [dir_watcher.py:finish():388] scan: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files
+2024-02-08 18:53:30,520 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/config.yaml config.yaml
+2024-02-08 18:53:30,520 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/requirements.txt requirements.txt
+2024-02-08 18:53:30,520 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/conda-environment.yaml conda-environment.yaml
+2024-02-08 18:53:30,520 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/wandb-metadata.json wandb-metadata.json
+2024-02-08 18:53:30,520 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/output.log output.log
+2024-02-08 18:53:30,522 INFO    SenderThread:1516 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/wandb-summary.json wandb-summary.json
+2024-02-08 18:53:30,524 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 10
+2024-02-08 18:53:30,524 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:30,524 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 10
+2024-02-08 18:53:30,525 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:30,525 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-02-08 18:53:30,525 INFO    SenderThread:1516 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:53:30,748 INFO    wandb-upload_0:1516 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/config.yaml
+2024-02-08 18:53:30,807 INFO    wandb-upload_1:1516 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/requirements.txt
+2024-02-08 18:53:30,838 INFO    wandb-upload_4:1516 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/wandb-summary.json
+2024-02-08 18:53:30,856 INFO    wandb-upload_3:1516 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/output.log
+2024-02-08 18:53:30,857 INFO    wandb-upload_2:1516 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/files/conda-environment.yaml
+2024-02-08 18:53:30,895 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:53:30,895 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:53:31,057 INFO    Thread-11 (_thread_body):1516 [sender.py:transition_state():617] send defer: 11
+2024-02-08 18:53:31,057 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:31,057 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 11
+2024-02-08 18:53:31,058 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:31,058 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-02-08 18:53:31,058 INFO    SenderThread:1516 [file_pusher.py:join():181] waiting for file pusher
+2024-02-08 18:53:31,058 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 12
+2024-02-08 18:53:31,058 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:31,058 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 12
+2024-02-08 18:53:31,058 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:31,058 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-02-08 18:53:31,058 INFO    SenderThread:1516 [file_stream.py:finish():595] file stream finish called
+2024-02-08 18:53:31,123 INFO    SenderThread:1516 [file_stream.py:finish():599] file stream finish is done
+2024-02-08 18:53:31,123 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 13
+2024-02-08 18:53:31,124 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:31,124 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 13
+2024-02-08 18:53:31,124 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:31,124 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-02-08 18:53:31,124 INFO    SenderThread:1516 [sender.py:transition_state():617] send defer: 14
+2024-02-08 18:53:31,124 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:53:31,124 INFO    HandlerThread:1516 [handler.py:handle_request_defer():172] handle defer: 14
+2024-02-08 18:53:31,124 DEBUG   SenderThread:1516 [sender.py:send():382] send: final
+2024-02-08 18:53:31,124 DEBUG   SenderThread:1516 [sender.py:send():382] send: footer
+2024-02-08 18:53:31,124 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:53:31,124 INFO    SenderThread:1516 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-02-08 18:53:31,125 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:53:31,125 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:53:31,126 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:53:31,126 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: server_info
+2024-02-08 18:53:31,126 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:53:31,126 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: server_info
+2024-02-08 18:53:31,128 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: get_summary
+2024-02-08 18:53:31,128 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: sampled_history
+2024-02-08 18:53:31,129 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:53:31,129 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: job_info
+2024-02-08 18:53:31,179 DEBUG   SenderThread:1516 [sender.py:send_request():409] send_request: job_info
+2024-02-08 18:53:31,179 INFO    MainThread:1516 [wandb_run.py:_footer_history_summary_info():3837] rendering history
+2024-02-08 18:53:31,180 INFO    MainThread:1516 [wandb_run.py:_footer_history_summary_info():3869] rendering summary
+2024-02-08 18:53:31,180 INFO    MainThread:1516 [wandb_run.py:_footer_sync_info():3796] logging synced files
+2024-02-08 18:53:31,180 DEBUG   HandlerThread:1516 [handler.py:handle_request():146] handle_request: shutdown
+2024-02-08 18:53:31,180 INFO    HandlerThread:1516 [handler.py:finish():866] shutting down handler
+2024-02-08 18:53:32,129 INFO    WriterThread:1516 [datastore.py:close():294] close: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/run-5uym8l7w.wandb
+2024-02-08 18:53:32,179 INFO    SenderThread:1516 [sender.py:finish():1548] shutting down sender
+2024-02-08 18:53:32,180 INFO    SenderThread:1516 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:53:32,180 INFO    SenderThread:1516 [file_pusher.py:join():181] waiting for file pusher

wandb/debug.log

@@ -0,0 +1,30 @@
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Configure stats pid to 1281
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/.config/wandb/settings
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/output-7b-26k-lora/wandb/settings
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-08 18:53:11,293 WARNING MainThread:1281 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/home/sagemaker-user/lora_finetuning_push_to_hub_save_local_latest.py', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py'}
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:_log_setup():524] Logging user logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/logs/debug.log
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:_log_setup():525] Logging internal logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_185311-5uym8l7w/logs/debug-internal.log
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:init():564] calling init triggers
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+config: {}
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:init():614] starting backend
+2024-02-08 18:53:11,293 INFO    MainThread:1281 [wandb_init.py:init():618] setting up manager
+2024-02-08 18:53:11,294 INFO    MainThread:1281 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-08 18:53:11,295 INFO    MainThread:1281 [wandb_init.py:init():624] backend started and connected
+2024-02-08 18:53:11,297 INFO    MainThread:1281 [wandb_init.py:init():716] updated telemetry
+2024-02-08 18:53:11,298 INFO    MainThread:1281 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-08 18:53:11,519 INFO    MainThread:1281 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-08 18:53:11,559 INFO    MainThread:1281 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.3 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-02-08 18:53:11,559 INFO    MainThread:1281 [wandb_init.py:init():800] starting run threads in backend
+2024-02-08 18:53:26,008 INFO    MainThread:1281 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-08 18:53:26,009 INFO    MainThread:1281 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-08 18:53:26,009 INFO    MainThread:1281 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-08 18:53:26,009 INFO    MainThread:1281 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-08 18:53:26,010 INFO    MainThread:1281 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-08 18:53:26,012 INFO    MainThread:1281 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 102400, 'max_position_embeddings': 4096, 'hidden_size': 4096, 'intermediate_size': 11008, 'num_hidden_layers': 30, 'num_attention_heads': 32, 'num_key_value_heads': 32, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_scaling': None, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 100000, 'pad_token_id': 0, 'eos_token_id': 100015, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'deepseek-ai/deepseek-coder-7b-instruct-v1.5', 'transformers_version': '4.31.0', 'attention_bias': False, 'model_type': 'llama', 'rope_theta': 10000.0, 'output_dir': 'output-7b-26k-lora', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'output-7b-26k-lora/runs/Feb08_18-53-01_default', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': 100, 'save_safetensors': False, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'ddp_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 4, 'past_index': -1, 'run_name': 'output-7b-26k-lora', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'ddp_broadcast_buffers': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'zzz99/output-7b-26k-lora', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'xpu_backend': 'None', 'cache_dir': 'None', 'model_max_length': 960, 'gradient_checkpointing_kwargs': "{'use_reentrant': False}", 'train_batch_size': 4, 'eval_batch_size': 1}
+2024-02-08 18:53:32,292 WARNING MsgRouterThr:1281 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240208_175238-v53k76w9/files/conda-environment.yaml

@@ -0,0 +1,662 @@
+name: base
+channels:
+  - file:///tmp/local_conda_forge_artifacts
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - _py-xgboost-mutex=2.0=gpu_0
+  - absl-py=2.0.0=pyhd8ed1ab_0
+  - accelerate=0.21.0=pyhd8ed1ab_0
+  - aiobotocore=2.7.0=pyhd8ed1ab_1
+  - aiohttp=3.9.1=py310h2372a71_0
+  - aioitertools=0.11.0=pyhd8ed1ab_0
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - aiosqlite=0.19.0=pyhd8ed1ab_0
+  - alsa-lib=1.2.10=hd590300_0
+  - altair=5.2.0=pyhd8ed1ab_0
+  - amazon-codewhisperer-jupyterlab-ext=2.0.1=pyhd8ed1ab_0
+  - amazon-sagemaker-jupyter-scheduler=3.0.5=pyhd8ed1ab_0
+  - ansi2html=1.8.0=py310hff52083_2
+  - ansiwrap=0.8.4=py_0
+  - antlr-python-runtime=4.9.3=pyhd8ed1ab_1
+  - anyio=3.7.1=pyhd8ed1ab_0
+  - aom=3.5.0=h27087fc_0
+  - appdirs=1.4.4=pyh9f0ad1d_0
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - astroid=3.0.1=py310hff52083_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - astunparse=1.6.3=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - async-timeout=4.0.3=pyhd8ed1ab_0
+  - atk-1.0=2.38.0=hd4edc92_1
+  - attr=2.5.1=h166bdaf_1
+  - attrs=23.1.0=pyh71513ae_1
+  - autogluon=0.8.2=pyhd8ed1ab_4
+  - autogluon.common=0.8.2=pyhd8ed1ab_4
+  - autogluon.core=0.8.2=light_py310h2d11d36_6
+  - autogluon.features=0.8.2=pyhd8ed1ab_3
+  - autogluon.multimodal=0.8.2=pyha770c72_5
+  - autogluon.tabular=0.8.2=pyha770c72_4
+  - autogluon.timeseries=0.8.2=pyhd8ed1ab_4
+  - autopep8=2.0.4=pyhd8ed1ab_0
+  - autovizwidget=0.21.0=pyh1a96a4e_0
+  - aws-c-auth=0.7.3=h28f7589_1
+  - aws-c-cal=0.6.1=hc309b26_1
+  - aws-c-common=0.9.0=hd590300_0
+  - aws-c-compression=0.2.17=h4d4d85c_2
+  - aws-c-event-stream=0.3.1=h2e3709c_4
+  - aws-c-http=0.7.11=h00aa349_4
+  - aws-c-io=0.13.32=he9a53bd_1
+  - aws-c-mqtt=0.9.3=hb447be9_1
+  - aws-c-s3=0.3.14=hf3aad02_1
+  - aws-c-sdkutils=0.1.12=h4d4d85c_1
+  - aws-checksums=0.1.17=h4d4d85c_1
+  - aws-crt-cpp=0.21.0=hb942446_5
+  - aws-embedded-metrics=3.2.0=pyhd8ed1ab_0
+  - aws-glue-sessions=1.0.4=pyhd8ed1ab_0
+  - aws-sdk-cpp=1.10.57=h85b1a90_19
+  - babel=2.13.1=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.2=pyha770c72_0
+  - binaryornot=0.4.4=py_1
+  - black=23.11.0=py310hff52083_0
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - blinker=1.7.0=pyhd8ed1ab_0
+  - blosc=1.21.5=h0f2a231_0
+  - boltons=23.0.0=pyhd8ed1ab_0
+  - boto3=1.28.64=pyhd8ed1ab_0
+  - botocore=1.31.64=pyhd8ed1ab_0
+  - brotli=1.0.9=h166bdaf_9
+  - brotli-bin=1.0.9=h166bdaf_9
+  - brotli-python=1.0.9=py310hd8f1fbe_9
+  - brunsli=0.1=h9c3ff4c_0
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.23.0=hd590300_0
+  - c-blosc2=2.11.3=hb4ffafa_0
+  - ca-certificates=2023.11.17=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cachetools=5.3.2=pyhd8ed1ab_0
+  - cairo=1.18.0=h3faef2a_0
+  - catalogue=2.0.10=py310hff52083_0
+  - catboost=1.2.2=py310hff52083_0
+  - certifi=2023.11.17=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - chardet=5.2.0=py310hff52083_1
+  - charls=2.4.2=h59595ed_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - click=8.1.7=unix_pyh707e725_0
+  - cloudpathlib=0.16.0=pyhd8ed1ab_0
+  - cloudpickle=2.2.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - conda=23.11.0=py310hff52083_1
+  - conda-libmamba-solver=23.11.1=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - confection=0.1.4=py310h17c5347_0
+  - contextlib2=21.6.0=pyhd8ed1ab_0
+  - contourpy=1.2.0=py310hd41b1e2_0
+  - cookiecutter=2.5.0=pyhca7485f_0
+  - croniter=1.4.1=pyhd8ed1ab_0
+  - cryptography=41.0.7=py310hb8475ec_1
+  - cuda-version=11.8=h70ddcb2_2
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudnn=8.8.0.121=hcdd5f01_4
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - cymem=2.0.8=py310hc6cd4ac_1
+  - cyrus-sasl=2.1.27=h54b06d7_7
+  - cython-blis=0.7.10=py310h1f7b6fc_2
+  - cytoolz=0.12.2=py310h2372a71_1
+  - dash=2.14.2=pyhd8ed1ab_0
+  - dask-core=2023.12.0=pyhd8ed1ab_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - dataclasses-json=0.6.3=pyhd8ed1ab_0
+  - datasets=2.14.7=pyhd8ed1ab_0
+  - dav1d=1.2.1=hd590300_0
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.8.0=py310hc6cd4ac_1
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - deepmerge=1.1.0=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - dill=0.3.7=pyhd8ed1ab_0
+  - distributed=2023.12.0=pyhd8ed1ab_0
+  - distro=1.8.0=pyhd8ed1ab_0
+  - docker-pycreds=0.4.0=py_0
+  - docstring-to-markdown=0.13=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - et_xmlfile=1.1.0=pyhd8ed1ab_0
+  - evaluate=0.4.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.5.0=hcb278e6_1
+  - faiss=1.7.4=py310h9ed8947_0_cpu
+  - faiss-cpu=1.7.4=h718b53a_0
+  - fastai=2.7.13=pyhd8ed1ab_0
+  - fastcore=1.5.29=pyhd8ed1ab_0
+  - fastdownload=0.0.7=pyhd8ed1ab_0
+  - fastprogress=1.0.3=pyhd8ed1ab_0
+  - filelock=3.13.1=pyhd8ed1ab_0
+  - flake8=6.1.0=pyhd8ed1ab_0
+  - flask=3.0.0=pyhd8ed1ab_0
+  - flatbuffers=23.5.26=h59595ed_1
+  - fmt=10.1.1=h00ab1b0_1
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_1
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.46.0=py310h2372a71_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - frozenlist=1.4.0=py310h2372a71_1
+  - fsspec=2023.6.0=pyh1a96a4e_0
+  - future=0.18.3=pyhd8ed1ab_0
+  - gast=0.4.0=pyh9f0ad1d_0
+  - gdk-pixbuf=2.42.10=h6b639ba_2
+  - gdown=4.7.1=pyhd8ed1ab_0
+  - gettext=0.21.1=h27087fc_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h0b41bf4_3
+  - gitdb=4.0.11=pyhd8ed1ab_0
+  - gitpython=3.1.40=pyhd8ed1ab_0
+  - glib=2.78.3=hfc55251_0
+  - glib-tools=2.78.3=hfc55251_0
+  - glog=0.6.0=h6f12383_0
+  - gluonts=0.13.7=pyhd8ed1ab_0
+  - gmp=6.3.0=h59595ed_0
+  - gmpy2=2.1.2=py310h3ec546c_1
+  - google-api-core=2.15.0=pyhd8ed1ab_0
+  - google-auth=2.25.2=pyhca7485f_0
+  - google-auth-oauthlib=1.0.0=pyhd8ed1ab_1
+  - google-pasta=0.2.0=pyh8c360ce_0
+  - googleapis-common-protos=1.62.0=pyhd8ed1ab_0
+  - graphite2=1.3.13=h58526e2_1001
+  - graphviz=8.1.0=h28d9a01_0
+  - grpcio=1.54.3=py310heca2aa9_0
+  - gst-plugins-base=1.22.7=h8e1006c_1
+  - gstreamer=1.22.7=h98fc4e7_1
+  - gtk2=2.24.33=h90689f9_2
+  - gts=0.7.6=h977cf35_4
+  - h5py=3.10.0=nompi_py310ha2ad45a_100
+  - harfbuzz=8.3.0=h3d44ed6_0
+  - hdf5=1.14.2=nompi_h4f84152_100
+  - hdijupyterutils=0.21.0=pyh1a96a4e_0
+  - huggingface_hub=0.19.4=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - imagecodecs=2023.8.12=py310hc929067_0
+  - imageio=2.31.5=pyh8c1a49c_0
+  - importlib-metadata=6.10.0=pyha770c72_0
+  - importlib_metadata=6.10.0=hd8ed1ab_0
+  - importlib_resources=6.1.1=pyhd8ed1ab_0
+  - iniconfig=2.0.0=pyhd8ed1ab_0
+  - ipykernel=6.26.0=pyhf8b6a83_0
+  - ipython=8.18.1=pyh707e725_3
+  - ipywidgets=8.1.1=pyhd8ed1ab_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - isort=5.12.0=pyhd8ed1ab_1
+  - itsdangerous=2.1.2=pyhd8ed1ab_0
+  - jax=0.4.20=pyhd8ed1ab_0
+  - jaxlib=0.4.14=cpu_py310h67d73b5_1
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.3.2=pyhd8ed1ab_0
+  - json5=0.9.14=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpath-ng=1.6.0=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.17.3=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.17.3=pyhd8ed1ab_0
+  - jupyter=1.0.0=pyhd8ed1ab_10
+  - jupyter-ai=2.6.0=pyhd8ed1ab_2
+  - jupyter-ai-magics=2.6.0=pyhd8ed1ab_1
+  - jupyter-dash=0.4.2=pyhd8ed1ab_1
+  - jupyter-lsp=2.2.1=pyhd8ed1ab_0
+  - jupyter-server-mathjax=0.2.6=pyh5bfe37b_1
+  - jupyter-server-proxy=4.1.0=pyhd8ed1ab_0
+  - jupyter_client=8.6.0=pyhd8ed1ab_0
+  - jupyter_console=6.6.3=pyhd8ed1ab_0
+  - jupyter_core=5.5.0=py310hff52083_0
+  - jupyter_events=0.6.3=pyhd8ed1ab_1
+  - jupyter_scheduler=2.4.0=pyha770c72_1
+  - jupyter_server=2.10.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0
+  - jupyterlab=4.0.9=pyhd8ed1ab_0
+  - jupyterlab-git=0.50.0=pyhd8ed1ab_0
+  - jupyterlab-lsp=5.0.1=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0
+  - jupyterlab_server=2.24.0=pyhd8ed1ab_0
+  - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0
+  - jxrlib=1.1=h7f98852_2
+  - keras=2.12.0=pyhd8ed1ab_0
+  - keras-preprocessing=1.1.2=pyhd8ed1ab_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py310hd41b1e2_1
+  - lame=3.100=h166bdaf_1003
+  - langchain=0.0.318=pyhd8ed1ab_0
+  - langcodes=3.3.0=pyhd8ed1ab_0
+  - langsmith=0.0.69=pyhd8ed1ab_0
+  - lcms2=2.15=haa2dc70_1
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20230125.3=cxx17_h59595ed_0
+  - libaec=1.1.2=h59595ed_1
+  - libarchive=3.7.2=h039dbb9_0
+  - libarrow=12.0.1=hb87d912_8_cpu
+  - libavif=0.11.1=h8182462_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libbrotlicommon=1.0.9=h166bdaf_9
+  - libbrotlidec=1.0.9=h166bdaf_9
+  - libbrotlienc=1.0.9=h166bdaf_9
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=16_linux64_mkl
+  - libclang=15.0.7=default_hb11cfb5_4
+  - libclang13=15.0.7=default_ha2b6cf4_4
+  - libcrc32c=1.1.2=h9c3ff4c_0
+  - libcups=2.3.3=h4637d8d_4
+  - libcurl=8.4.0=hca28451_0
+  - libdeflate=1.18=h0b41bf4_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.5.0=hcb278e6_1
+  - libfaiss=1.7.4=hf47d654_0_cpu
+  - libfaiss-avx2=1.7.4=h1234567_0_cpu
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgcrypt=1.10.3=hd590300_0
+  - libgd=2.3.3=h74d50f4_7
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libglib=2.78.3=h783c2da_0
+  - libgoogle-cloud=2.12.0=hac9eb74_1
+  - libgpg-error=1.47=h71f35ed_0
+  - libgrpc=1.54.3=hb20ce57_0
+  - libhwloc=2.9.3=default_h554bfaf_1009
+  - libiconv=1.17=hd590300_1
+  - libjpeg-turbo=2.1.5.1=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libllvm14=14.0.6=hcd5def8_4
+  - libllvm15=15.0.7=h5cf9203_3
+  - libmagma=2.7.1=hc72dce7_6
+  - libmagma_sparse=2.7.1=h8354cda_6
+  - libmamba=1.5.3=had39da4_2
+  - libmambapy=1.5.3=py310h39ff949_2
+  - libnghttp2=1.58.0=h47da74e_0
+  - libnsl=2.0.1=hd590300_0
+  - libntlm=1.4=h7f98852_1002
+  - libnuma=2.0.16=h0b41bf4_1
+  - libogg=1.3.4=h7f98852_1
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.39=h753d276_0
+  - libpq=15.4=hfc447b1_2
+  - libprotobuf=3.21.12=hfc55251_2
+  - librsvg=2.56.3=h98fae49_0
+  - libsecret=0.18.8=h329b89f_2
+  - libsentencepiece=0.1.99=h180e1df_0
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libsystemd0=255=h3516f8a_0
+  - libthrift=0.18.1=h8fd135c_2
+  - libtiff=4.5.1=h8b53f26_1
+  - libtool=2.4.7=h27087fc_0
+  - libutf8proc=2.8.0=h166bdaf_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libuv=1.46.0=hd590300_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp=1.3.1=hbf2b3c1_0
+  - libwebp-base=1.3.1=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxgboost=1.7.6=cuda118_hd3b444d_6
+  - libxkbcommon=1.6.0=h5d7e998_0
+  - libxml2=2.11.6=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - libzopfli=1.0.3=h9c3ff4c_0
+  - lightgbm=3.3.5=py310heca2aa9_0
+  - lightning-utilities=0.10.0=pyhd8ed1ab_0
+  - llvm-openmp=17.0.6=h4dfa4b3_0
+  - llvmlite=0.41.1=py310h1b8f574_0
+  - locket=1.0.0=pyhd8ed1ab_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - magma=2.7.1=ha770c72_6
+  - markdown=3.5.1=pyhd8ed1ab_0
+  - markdown-it-py=3.0.0=pyhd8ed1ab_0
+  - markupsafe=2.1.3=py310h2372a71_1
+  - marshmallow=3.20.1=pyhd8ed1ab_0
+  - matplotlib=3.8.2=py310hff52083_0
+  - matplotlib-base=3.8.2=py310h62c0568_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mccabe=0.7.0=pyhd8ed1ab_0
+  - mdurl=0.1.0=pyhd8ed1ab_0
+  - menuinst=2.0.1=py310hff52083_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2022.2.1=h84fe81f_16997
+  - ml_dtypes=0.3.1=py310hcc13569_2
+  - mlforecast=0.7.3=pyhd8ed1ab_0
+  - mock=5.1.0=pyhd8ed1ab_0
+  - model-index=0.1.11=pyhd8ed1ab_0
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h9458935_0
+  - mpg123=1.32.3=h59595ed_0
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - msgpack-python=1.0.7=py310hd41b1e2_0
+  - multidict=6.0.4=py310h2372a71_1
+  - multiprocess=0.70.15=py310h2372a71_1
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.10=py310hc6cd4ac_1
+  - mypy_extensions=1.0.0=pyha770c72_0
+  - mysql-common=8.0.33=hf1915f5_6
+  - mysql-libs=8.0.33=hca2cd23_6
+  - nbclient=0.8.0=pyhd8ed1ab_0
+  - nbconvert=7.12.0=pyhd8ed1ab_0
+  - nbconvert-core=7.12.0=pyhd8ed1ab_0
+  - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0
+  - nbdime=4.0.1=pyhd8ed1ab_0
+  - nbformat=5.9.2=pyhd8ed1ab_0
+  - nccl=2.19.4.1=h6103f9b_0
+  - ncurses=6.4=h59595ed_2
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - networkx=3.2.1=pyhd8ed1ab_0
+  - nlpaug=1.1.11=pyhd8ed1ab_1
+  - nltk=3.8.1=pyhd8ed1ab_0
+  - nodejs=18.18.2=hb753e55_1
+  - nose=1.3.7=py_1006
+  - notebook=7.0.6=pyhd8ed1ab_0
+  - notebook-shim=0.2.3=pyhd8ed1ab_0
+  - nptyping=2.4.1=pyhd8ed1ab_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.95=h1d7d5a4_0
+  - numba=0.58.1=py310h7dc5dd1_0
+  - numexpr=2.8.7=mkl_py310h5e07f1c_4
+  - numpy=1.26.2=py310hb13e2d6_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - omegaconf=2.2.3=pyhd8ed1ab_0
+  - openai=0.28.1=pyhd8ed1ab_0
+  - openapi-schema-pydantic=1.2.4=pyhd8ed1ab_0
+  - openjpeg=2.5.0=hfec8fc6_2
+  - openmim=0.3.7=pyhd8ed1ab_0
+  - openpyxl=3.1.2=py310h2372a71_1
+  - openssl=3.1.4=hd590300_0
+  - opt-einsum=3.3.0=hd8ed1ab_2
+  - opt_einsum=3.3.0=pyhc1e730c_2
+  - orc=1.9.0=h2f23424_1
+  - ordered-set=4.1.0=pyhd8ed1ab_0
+  - overrides=7.4.0=pyhd8ed1ab_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pandas=2.1.4=py310hcc13569_0
+  - pandas-stubs=2.1.1.230928=pyhd8ed1ab_1
+  - pandoc=3.1.3=h32600fe_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - pango=1.50.14=ha41ecd1_2
+  - papermill=2.4.0=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - partd=1.4.1=pyhd8ed1ab_0
+  - pathos=0.3.1=pyhd8ed1ab_0
+  - pathspec=0.12.1=pyhd8ed1ab_0
+  - pathtools=0.1.2=py_1
+  - pathy=0.10.2=pyhd8ed1ab_0
+  - patsy=0.5.4=pyhd8ed1ab_0
+  - pcre2=10.42=hcad00b1_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=9.5.0=py310h582fbeb_1
+  - pip=23.3.1=pyhd8ed1ab_0
+  - pixman=0.42.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - plotly=5.18.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - ply=3.11=py_1
+  - pox=0.3.3=pyhd8ed1ab_0
+  - ppft=1.7.6.7=pyhd8ed1ab_0
+  - preshed=3.0.9=py310hc6cd4ac_1
+  - prometheus_client=0.19.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.41=pyha770c72_0
+  - prompt_toolkit=3.0.41=hd8ed1ab_0
+  - protobuf=4.21.12=py310heca2aa9_0
+  - psutil=5.9.5=py310h2372a71_1
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio-client=16.1=hb77b528_5
+  - pure-sasl=0.6.2=pyhd8ed1ab_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - py-xgboost=1.7.6=cuda118_py310h9840055_6
+  - py-xgboost-gpu=1.7.6=py310h9840055_6
+  - pyarrow=12.0.1=py310h0576679_8_cpu
+  - pyarrow-hotfix=0.6=pyhd8ed1ab_0
+  - pyasn1=0.5.1=pyhd8ed1ab_0
+  - pyasn1-modules=0.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycodestyle=2.11.1=pyhd8ed1ab_0
+  - pycosat=0.6.6=py310h2372a71_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydocstyle=6.3.0=pyhd8ed1ab_0
+  - pyflakes=3.1.0=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pyhive=0.7.0=pyhd8ed1ab_0
+  - pyjwt=2.8.0=pyhd8ed1ab_0
+  - pylint=3.0.3=pyhd8ed1ab_0
+  - pyopenssl=23.3.0=pyhd8ed1ab_0
+  - pyparsing=3.1.1=pyhd8ed1ab_0
+  - pyqt=5.15.9=py310h04931ad_5
+  - pyqt5-sip=12.12.2=py310hc6cd4ac_5
+  - pyrsistent=0.20.0=py310h2372a71_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - pyspnego=0.9.1=py310h2372a71_2
+  - pytesseract=0.3.10=pyhd8ed1ab_0
+  - pytest=7.4.3=pyhd8ed1ab_0
+  - pytest-subtests=0.11.0=pyhd8ed1ab_0
+  - python=3.10.13=hd12c33a_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.0=pyhd8ed1ab_0
+  - python-flatbuffers=23.5.26=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=pyh22cad53_0
+  - python-gssapi=1.8.3=py310h98feede_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python-lsp-jsonrpc=1.1.2=pyhd8ed1ab_0
+  - python-lsp-server=1.9.0=pyhd8ed1ab_0
+  - python-lsp-server-base=1.9.0=pyhd8ed1ab_0
+  - python-slugify=8.0.1=pyhd8ed1ab_2
+  - python-tzdata=2023.3=pyhd8ed1ab_0
+  - python-xxhash=3.4.1=py310h2372a71_0
+  - python_abi=3.10=4_cp310
+  - pytoolconfig=1.2.5=pyhd8ed1ab_0
+  - pytorch=2.0.0=cuda112py310he33e0d6_200
+  - pytorch-gpu=2.0.0=cuda112py310h9871d0b_200
+  - pytorch-lightning=2.0.9=pyhd8ed1ab_0
+  - pytorch-metric-learning=1.7.3=pyhd8ed1ab_0
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywavelets=1.4.1=py310h1f7b6fc_1
+  - pyyaml=6.0.1=py310h2372a71_1
+  - pyzmq=25.1.2=py310h795f18f_0
+  - qt-main=5.15.8=hc47bfe8_16
+  - qtconsole-base=5.5.1=pyha770c72_0
+  - qtpy=2.4.1=pyhd8ed1ab_0
+  - rdma-core=28.9=h59595ed_1
+  - re2=2023.03.02=h8c504da_0
+  - readline=8.2=h8228510_1
+  - regex=2023.10.3=py310h2372a71_0
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - requests-kerberos=0.14.0=pyhd8ed1ab_1
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - retrying=1.3.3=py_2
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rich=13.7.0=pyhd8ed1ab_0
+  - ripgrep=14.0.3=he8a937b_1
+  - rope=1.11.0=pyhd8ed1ab_1
+  - rsa=4.9=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py310h2372a71_0
+  - ruamel.yaml.clib=0.2.7=py310h2372a71_2
+  - s2n=1.3.49=h06160fa_0
+  - s3transfer=0.7.0=pyhd8ed1ab_1
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - safetensors=0.3.3=py310hcb5633a_1
+  - sagemaker-code-editor=1.83.1=ha1e3dc1_3
+  - sagemaker-headless-execution-driver=0.0.12=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-emr-extension=0.1.9=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension=0.2.0=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension-common=0.1.9=pyhd8ed1ab_0
+  - sagemaker-kernel-wrapper=0.0.2=pyhd8ed1ab_0
+  - sagemaker-python-sdk=2.198.0=pyhd8ed1ab_0
+  - sagemaker-studio-analytics-extension=0.0.21=pyhd8ed1ab_0
+  - sagemaker-studio-sparkmagic-lib=0.1.4=pyhd8ed1ab_0
+  - sasl=0.3.1=py310hd8f1fbe_2
+  - schema=0.7.5=pyhd8ed1ab_1
+  - scikit-image=0.19.3=py310h769672d_2
+  - scikit-learn=1.3.2=py310h1fdf081_2
+  - scipy=1.11.4=py310hb13e2d6_0
+  - send2trash=1.8.2=pyh41d4057_0
+  - sentencepiece=0.1.99=hff52083_0
+  - sentencepiece-python=0.1.99=py310hebdb9f0_0
+  - sentencepiece-spm=0.1.99=h180e1df_0
+  - sentry-sdk=1.38.0=pyhd8ed1ab_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setproctitle=1.3.3=py310h2372a71_0
+  - setuptools=68.2.2=pyhd8ed1ab_0
+  - shellingham=1.5.4=pyhd8ed1ab_0
+  - simpervisor=1.0.0=pyhd8ed1ab_0
+  - sip=6.7.12=py310hc6cd4ac_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sleef=3.5.1=h9b69904_2
+  - smart_open=5.2.1=pyhd8ed1ab_0
+  - smdebug-rulesconfig=1.0.1=pyhd3deb0d_1
+  - smmap=5.0.0=pyhd8ed1ab_0
+  - snappy=1.1.10=h9fff704_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - snowballstemmer=2.2.0=pyhd8ed1ab_0
+  - sortedcontainers=2.4.0=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - spacy=3.7.2=py310hcb52e73_0
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.5=pyhd8ed1ab_0
+  - sparkmagic=0.21.0=pyhd8ed1ab_0
+  - srsly=2.4.8=py310hc6cd4ac_1
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - statsforecast=1.4.0=pyhd8ed1ab_0
+  - statsmodels=0.14.0=py310h1f7b6fc_3
+  - supervisor=4.2.5=py310hff52083_2
+  - sympy=1.12=pypyh9d50eac_103
+  - tabulate=0.9.0=pyhd8ed1ab_1
+  - tbb=2021.11.0=h00ab1b0_0
+  - tblib=1.7.0=pyhd8ed1ab_0
+  - tenacity=8.2.3=pyhd8ed1ab_0
+  - tensorboard=2.12.3=pyhd8ed1ab_0
+  - tensorboard-data-server=0.7.0=py310h75e40e8_1
+  - tensorflow=2.12.1=cuda112py310h457873b_1
+  - tensorflow-base=2.12.1=cuda112py310h1d65785_1
+  - tensorflow-estimator=2.12.1=cuda112py310ha5e6de5_1
+  - termcolor=2.3.0=pyhd8ed1ab_0
+  - terminado=0.18.0=pyh0d859eb_0
+  - text-unidecode=1.3=pyhd8ed1ab_1
+  - textwrap3=0.9.2=py_0
+  - thinc=8.2.1=py310hcb52e73_0
+  - threadpoolctl=3.2.0=pyha21a80b_0
+  - thrift=0.19.0=py310hc6cd4ac_1
+  - thrift_sasl=0.4.3=pyhd8ed1ab_2
+  - tifffile=2023.12.9=pyhd8ed1ab_0
+  - tiktoken=0.5.2=py310h47bb294_0
+  - timm=0.9.12=pyhd8ed1ab_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tokenizers=0.13.3=py310he1f1126_0
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tomlkit=0.12.3=pyha770c72_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchmetrics=1.0.3=pyhd8ed1ab_0
+  - torchvision=0.15.2=cuda112py310h0801bf5_1
+  - tornado=6.3.3=py310h2372a71_1
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - traitlets=5.14.0=pyhd8ed1ab_0
+  - transformers=4.31.0=pyhd8ed1ab_0
+  - truststore=0.8.0=pyhd8ed1ab_0
+  - typer=0.9.0=pyhd8ed1ab_0
+  - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0
+  - types-pytz=2023.3.1.1=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - typing_inspect=0.9.0=pyhd8ed1ab_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - typish=1.9.3=pyhd8ed1ab_0
+  - tzdata=2023c=h71feb2d_0
+  - ucx=1.14.1=h64cca9d_5
+  - ujson=5.9.0=py310hc6cd4ac_0
+  - unicodedata2=15.1.0=py310h2372a71_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - urllib3=1.26.18=pyhd8ed1ab_0
+  - wandb=0.16.1=pyhd8ed1ab_0
+  - wasabi=1.1.2=py310hff52083_0
+  - wcwidth=0.2.12=pyhd8ed1ab_0
+  - weasel=0.3.4=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.7.0=pyhd8ed1ab_0
+  - werkzeug=3.0.1=pyhd8ed1ab_0
+  - whatthepatch=1.0.5=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - widgetsnbextension=4.0.9=pyhd8ed1ab_0
+  - window-ops=0.0.14=pyhd8ed1ab_0
+  - wrapt=1.16.0=py310h2372a71_0
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xgboost=1.7.6=cuda118_py310h9840055_6
+  - xkeyboard-config=2.40=hd590300_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.7=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xxhash=0.8.2=hd590300_0
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yaml-cpp=0.8.0=h59595ed_0
+  - yapf=0.40.1=pyhd8ed1ab_0
+  - yarl=1.9.3=py310h2372a71_0
+  - zeromq=4.3.5=h59595ed_0
+  - zfp=1.0.0=h59595ed_4
+  - zict=3.0.0=pyhd8ed1ab_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zlib-ng=2.0.7=h0b41bf4_0
+  - zstandard=0.22.0=py310h1275a96_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - annotated-types==0.6.0
+      - greenlet==3.0.1
+      - jupyter-scheduler==2.4.0
+      - krb5==0.5.1
+      - peft==0.8.2
+      - pydantic==2.5.1
+      - pydantic-core==2.14.3
+      - sqlalchemy==1.4.50
+prefix: /opt/conda

wandb/run-20240208_175238-v53k76w9/files/config.yaml

@@ -0,0 +1,614 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.13
+    cli_version: 0.16.1
+    framework: huggingface
+    huggingface_version: 4.31.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1707414758.178795
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      3:
+      - 7
+      - 23
+      4: 3.10.13
+      5: 0.16.1
+      6: 4.31.0
+      8:
+      - 5
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+vocab_size:
+  desc: null
+  value: 102400
+max_position_embeddings:
+  desc: null
+  value: 4096
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 11008
+num_hidden_layers:
+  desc: null
+  value: 30
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 32
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_scaling:
+  desc: null
+  value: null
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 100000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 100015
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+transformers_version:
+  desc: null
+  value: 4.31.0
+attention_bias:
+  desc: null
+  value: false
+model_type:
+  desc: null
+  value: llama
+rope_theta:
+  desc: null
+  value: 10000.0
+output_dir:
+  desc: null
+  value: output-7b-26k-lora
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: epoch
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_device_eval_batch_size:
+  desc: null
+  value: 1
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 2.0e-05
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 2
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 10
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: output-7b-26k-lora/runs/Feb08_17-51-59_default
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_steps:
+  desc: null
+  value: 100
+save_total_limit:
+  desc: null
+  value: 100
+save_safetensors:
+  desc: null
+  value: false
+save_on_each_node:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: None
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: None
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: None
+tpu_num_cores:
+  desc: null
+  value: None
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: '[]'
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: None
+dataloader_num_workers:
+  desc: null
+  value: 4
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: output-7b-26k-lora
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: None
+load_best_model_at_end:
+  desc: null
+  value: true
+metric_for_best_model:
+  desc: null
+  value: loss
+greater_is_better:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+sharded_ddp:
+  desc: null
+  value: '[]'
+fsdp:
+  desc: null
+  value: '[]'
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value: '{''fsdp_min_num_params'': 0, ''xla'': False, ''xla_fsdp_grad_ckpt'': False}'
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: None
+deepspeed:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: None
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value: '[''tensorboard'', ''wandb'']'
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+ddp_broadcast_buffers:
+  desc: null
+  value: None
+dataloader_pin_memory:
+  desc: null
+  value: true
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: None
+hub_model_id:
+  desc: null
+  value: zzz99/output-7b-26k-lora
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: None
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: None
+torch_compile_mode:
+  desc: null
+  value: None
+xpu_backend:
+  desc: null
+  value: None
+cache_dir:
+  desc: null
+  value: None
+model_max_length:
+  desc: null
+  value: 960
+gradient_checkpointing_kwargs:
+  desc: null
+  value: '{''use_reentrant'': False}'
+train_batch_size:
+  desc: null
+  value: 4
+eval_batch_size:
+  desc: null
+  value: 1

wandb/run-20240208_175238-v53k76w9/files/output.log

@@ -0,0 +1,21 @@
+  0%|                                                                                                                              | 0/2 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+Traceback (most recent call last):
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py", line 236, in <module>
+    train()
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py", line 223, in train
+    trainer.train()
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+    return inner_training_loop(
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2665, in training_step
+    self.accelerator.backward(loss)
+  File "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py", line 1853, in backward
+    loss.backward(**kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20240208_175238-v53k76w9/files/requirements.txt

@@ -0,0 +1,419 @@
+absl-py==2.0.0
+accelerate==0.21.0
+aiobotocore==2.7.0
+aiohttp==3.9.1
+aioitertools==0.11.0
+aiosignal==1.3.1
+aiosqlite==0.19.0
+altair==5.2.0
+amazon-codewhisperer-jupyterlab-ext==2.0.1
+amazon-sagemaker-jupyter-scheduler==3.0.5
+annotated-types==0.6.0
+ansi2html==0.0.0
+ansiwrap==0.8.4
+antlr4-python3-runtime==4.9.3
+anyio==3.7.1
+appdirs==1.4.4
+archspec==0.2.2
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+astroid==3.0.1
+asttokens==2.4.1
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+autogluon.common==0.8.2
+autogluon.core==0.8.2
+autogluon.features==0.8.2
+autogluon.multimodal==0.8.2
+autogluon.tabular==0.8.2
+autogluon.timeseries==0.8.2
+autogluon==0.8.2
+autopep8==2.0.4
+autovizwidget==0.21.0
+aws-embedded-metrics==3.2.0
+aws-glue-sessions==1.0.4
+babel==2.13.1
+beautifulsoup4==4.12.2
+binaryornot==0.4.4
+black==23.11.0
+bleach==6.1.0
+blinker==1.7.0
+blis==0.7.10
+boltons==23.0.0
+boto3==1.28.64
+botocore==1.31.64
+brotli==1.0.9
+cached-property==1.5.2
+cachetools==5.3.2
+catalogue==2.0.10
+catboost==1.2.2
+certifi==2023.11.17
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==2.2.1
+colorama==0.4.6
+comm==0.1.4
+conda-libmamba-solver==23.11.1
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.11.0
+confection==0.1.4
+contextlib2==21.6.0
+contourpy==1.2.0
+cookiecutter==2.5.0
+croniter==1.4.1
+cryptography==41.0.7
+cycler==0.12.1
+cymem==2.0.8
+cytoolz==0.12.2
+dash==2.14.2
+dask==2023.12.0
+dataclasses-json==0.6.3
+dataclasses==0.8
+datasets==2.14.7
+debugpy==1.8.0
+decorator==5.1.1
+deepmerge==1.1.0
+defusedxml==0.7.1
+dill==0.3.7
+distributed==2023.12.0
+distro==1.8.0
+docker-pycreds==0.4.0
+docstring-to-markdown==0.13
+entrypoints==0.4
+et-xmlfile==1.1.0
+evaluate==0.4.1
+exceptiongroup==1.2.0
+executing==2.0.1
+faiss==1.7.4
+fastai==2.7.13
+fastcore==1.5.29
+fastdownload==0.0.7
+fastjsonschema==2.19.0
+fastprogress==1.0.3
+filelock==3.13.1
+flake8==6.1.0
+flask==3.0.0
+flatbuffers==23.5.26
+fonttools==4.46.0
+fqdn==1.5.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+future==0.18.3
+gast==0.4.0
+gdown==4.7.1
+gitdb==4.0.11
+gitpython==3.1.40
+gluonts==0.13.7
+gmpy2==2.1.2
+google-api-core==2.15.0
+google-auth-oauthlib==1.0.0
+google-auth==2.25.2
+google-pasta==0.2.0
+googleapis-common-protos==1.62.0
+graphviz==0.20.1
+greenlet==3.0.1
+grpcio==1.54.3
+gssapi==1.8.3
+h5py==3.10.0
+hdijupyterutils==0.21.0
+huggingface-hub==0.19.4
+idna==3.6
+imagecodecs==2023.8.12
+imageio==2.31.5
+importlib-metadata==6.10.0
+importlib-resources==6.1.1
+iniconfig==2.0.0
+ipykernel==6.26.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+isort==5.12.0
+itsdangerous==2.1.2
+jax==0.4.20
+jaxlib==0.4.14
+jedi==0.19.1
+jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpath-ng==1.6.0
+jsonpointer==2.4
+jsonschema==4.17.3
+jupyter-ai-magics==2.6.0
+jupyter-ai==2.6.0
+jupyter-client==8.6.0
+jupyter-console==6.6.3
+jupyter-core==5.5.0
+jupyter-dash==0.4.2
+jupyter-events==0.6.3
+jupyter-lsp==2.2.1
+jupyter-scheduler==2.4.0
+jupyter-server-mathjax==0.2.6
+jupyter-server-proxy==4.1.0
+jupyter-server-terminals==0.5.0
+jupyter-server==2.10.0
+jupyter==1.0.0
+jupyterlab-git==0.50.0
+jupyterlab-lsp==5.0.1
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.24.0
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+keras-preprocessing==1.1.2
+keras==2.12.0
+kiwisolver==1.4.5
+krb5==0.5.1
+langchain==0.0.318
+langcodes==3.3.0
+langsmith==0.0.69
+libmambapy==1.5.3
+lightgbm==3.3.5
+lightning-utilities==0.10.0
+llvmlite==0.41.1
+locket==1.0.0
+markdown-it-py==3.0.0
+markdown==3.5.1
+markupsafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mccabe==0.7.0
+mdurl==0.1.0
+menuinst==2.0.1
+mistune==3.0.2
+ml-dtypes==0.3.1
+mlforecast==0.7.3
+mock==5.1.0
+model-index==0.1.11
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+munkres==1.1.4
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.12.0
+nbdime==4.0.1
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nlpaug==1.1.11
+nltk==3.8.1
+nose==1.3.7
+notebook-shim==0.2.3
+notebook==7.0.6
+nptyping==2.4.1
+numba==0.58.1
+numexpr==2.8.7
+numpy==1.26.2
+oauthlib==3.2.2
+omegaconf==2.2.3
+openai==0.28.1
+openapi-schema-pydantic==1.2.4
+openmim==0.3.7
+openpyxl==3.1.2
+opt-einsum==3.3.0
+ordered-set==4.1.0
+overrides==7.4.0
+packaging==23.2
+pandas-stubs==2.1.1.230928
+pandas==2.1.4
+pandocfilters==1.5.0
+papermill==2.4.0
+parso==0.8.3
+partd==1.4.1
+pathos==0.3.1
+pathspec==0.12.1
+pathtools==0.1.2
+pathy==0.10.2
+patsy==0.5.4
+peft==0.8.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.5.0
+pip==23.3.1
+pkgutil-resolve-name==1.3.10
+platformdirs==4.1.0
+plotly==5.18.0
+pluggy==1.3.0
+ply==3.11
+pox==0.3.3
+ppft==1.7.6.7
+preshed==3.0.9
+prometheus-client==0.19.0
+prompt-toolkit==3.0.41
+protobuf==4.21.12
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pure-sasl==0.6.2
+pyarrow-hotfix==0.6
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pycodestyle==2.11.1
+pycosat==0.6.6
+pycparser==2.21
+pydantic-core==2.14.3
+pydantic==1.10.13
+pydocstyle==6.3.0
+pyflakes==3.1.0
+pygments==2.17.2
+pyhive==0.7.0
+pyjwt==2.8.0
+pylint==3.0.3
+pyopenssl==23.3.0
+pyparsing==3.1.1
+pyqt5-sip==12.12.2
+pyqt5==5.15.9
+pyrsistent==0.20.0
+pysocks==1.7.1
+pyspnego==0.9.1
+pytesseract==0.3.10
+pytest-subtests==0.11.0
+pytest==7.4.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.9.0
+python-slugify==8.0.1
+pytoolconfig==1.2.5
+pytorch-lightning==2.0.9
+pytorch-metric-learning==1.7.3
+pytz==2023.3
+pyu2f==0.1.5
+pywavelets==1.4.1
+pyyaml==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+qtpy==2.4.1
+regex==2023.10.3
+requests-kerberos==0.14.0
+requests-oauthlib==1.3.1
+requests==2.31.0
+responses==0.18.0
+retrying==1.3.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rope==1.11.0
+rsa==4.9
+ruamel.yaml.clib==0.2.7
+ruamel.yaml==0.18.5
+s3transfer==0.7.0
+sacremoses==0.0.53
+safetensors==0.3.3
+sagemaker-headless-execution-driver==0.0.12
+sagemaker-jupyterlab-emr-extension==0.1.9
+sagemaker-jupyterlab-extension-common==0.1.9
+sagemaker-jupyterlab-extension==0.2.0
+sagemaker-kernel-wrapper==0.0.2
+sagemaker-studio-analytics-extension==0.0.21
+sagemaker-studio-sparkmagic-lib==0.1.4
+sagemaker==2.198.0
+sasl==0.3.1
+schema==0.7.5
+scikit-image==0.19.3
+scikit-learn==1.3.2
+scipy==1.11.4
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==1.38.0
+seqeval==1.2.2
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+simpervisor==1.0.0
+sip==6.7.12
+six==1.16.0
+smart-open==5.2.1
+smdebug-rulesconfig==1.0.1
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sparkmagic==0.21.0
+sqlalchemy==1.4.49
+srsly==2.4.8
+stack-data==0.6.2
+statsforecast==1.4.0
+statsmodels==0.14.0
+supervisor==4.2.5
+sympy==1.12
+tabulate==0.9.0
+tblib==1.7.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.0
+tensorboard==2.12.3
+tensorflow-estimator==2.12.0
+tensorflow==2.12.1
+termcolor==2.3.0
+terminado==0.18.0
+text-unidecode==1.3
+textwrap3==0.9.2
+thinc==8.2.1
+threadpoolctl==3.2.0
+thrift-sasl==0.4.3
+thrift==0.19.0
+tifffile==2023.12.9
+tiktoken==0.5.2
+timm==0.9.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.12.3
+toolz==0.12.0
+torch==2.0.0.post200
+torchmetrics==1.0.3
+torchvision==0.15.2a0+072ec57
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.31.0
+truststore==0.8.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+types-pytz==2023.3.1.1
+typing-extensions==4.5.0
+typing-inspect==0.9.0
+typing-utils==0.1.0
+typish==1.9.3
+tzdata==2023.3
+ujson==5.9.0
+unicodedata2==15.1.0
+uri-template==1.3.0
+urllib3==1.26.18
+wandb==0.16.1
+wasabi==1.1.2
+wcwidth==0.2.12
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.1
+whatthepatch==1.0.5
+wheel==0.42.0
+widgetsnbextension==4.0.9
+window-ops==0.0.14
+wrapt==1.16.0
+xgboost==1.7.6
+xxhash==3.4.1
+yapf==0.40.1
+yarl==1.9.3
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.22.0

wandb/run-20240208_175238-v53k76w9/files/wandb-metadata.json

@@ -0,0 +1,1029 @@
+{
+    "os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
+    "python": "3.10.13",
+    "heartbeatAt": "2024-02-08T17:52:38.631590",
+    "startedAt": "2024-02-08T17:52:38.174980",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py",
+    "codePathLocal": null,
+    "host": "default",
+    "username": "sagemaker-user",
+    "executable": "/opt/conda/bin/python3",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 3096.191276041665,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2806.797,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2305.192,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2429.072,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2448.527,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2217.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2733.2,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2599.219,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2830.092,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2856.656,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2766.239,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2761.423,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2600.369,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2658.209,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2747.075,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.035,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2742.13,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2818.903,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2743.213,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2432.09,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2731.02,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2808.377,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2777.618,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2290.979,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2230.543,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2738.423,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2903.95,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2970.61,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.839,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2689.335,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2791.925,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2731.728,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2813.357,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2794.296,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2747.123,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2795.435,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2767.017,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2722.071,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.527,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2932.725,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3292.093,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.824,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3256.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3256.429,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3259.575,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2700.636,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3234.186,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3206.966,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.085,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3282.893,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3279.04,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3278.154,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3283.989,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2562.18,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2954.006,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2762.278,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3275.22,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.85,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3291.939,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2973.521,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2966.002,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2966.843,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2645.143,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3046.118,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3006.852,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.715,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2922.754,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2906.522,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3028.907,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2966.081,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2917.105,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.43,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.481,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3270.344,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2930.864,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2879.041,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2902.742,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.401,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2686.543,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3222.046,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.97,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.666,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2754.074,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.533,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2812.149,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.31,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.208,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2779.101,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.477,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2825.936,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2204.979,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2851.77,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2797.024,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2325.643,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2850.865,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2919.634,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2910.972,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2523.164,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2297.34,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2193.979,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2128.798,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1907.218,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2921.246,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2408.454,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2296.906,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2877.315,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2985.576,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2977.194,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2982.705,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2367.542,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2232.475,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2720.158,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2260.753,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2215.697,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2278.892,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2009.932,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2813.45,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2248.538,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2789.291,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2481.076,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2033.475,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2214.296,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2762.868,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2273.931,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2891.192,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2217.993,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2306.666,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2372.976,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2322.672,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2325.945,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2332.493,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2202.398,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2130.875,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2034.318,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2539.829,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2088.35,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2427.524,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2432.02,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2521.716,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3047.178,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2452.92,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2398.052,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2930.232,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2915.194,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3050.935,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2985.592,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2999.519,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2954.304,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3253.761,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2547.987,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2791.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2669.218,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3304.846,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3017.308,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.861,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2977.232,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2939.823,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.543,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3014.24,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.908,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3014.885,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.521,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.848,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.858,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.813,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2998.973,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.759,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3026.427,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.35,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2507.162,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3250.875,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.582,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.791,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2876.895,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.637,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.935,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.409,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.545,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2845.582,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.789,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3212.048,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2598.735,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.632,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.179,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.805,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.982,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2498.549,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.222,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.448,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2830.786,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.116,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.39,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.373,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 32.0,
+            "used": 0.012481689453125
+        }
+    },
+    "gpu": "NVIDIA A10G",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        }
+    ],
+    "memory": {
+        "total": 747.9597625732422
+    }
+}

wandb/run-20240208_175238-v53k76w9/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 17}}

wandb/run-20240208_175238-v53k76w9/logs/debug-internal.log

@@ -0,0 +1,194 @@
+2024-02-08 17:52:38,179 INFO    StreamThr :1317 [internal.py:wandb_internal():86] W&B internal server running at pid: 1317, started at: 2024-02-08 17:52:38.179167
+2024-02-08 17:52:38,184 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: status
+2024-02-08 17:52:38,185 INFO    WriterThread:1317 [datastore.py:open_for_write():85] open: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/run-v53k76w9.wandb
+2024-02-08 17:52:38,186 DEBUG   SenderThread:1317 [sender.py:send():382] send: header
+2024-02-08 17:52:38,186 DEBUG   SenderThread:1317 [sender.py:send():382] send: run
+2024-02-08 17:52:38,455 INFO    SenderThread:1317 [dir_watcher.py:__init__():211] watching files in: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files
+2024-02-08 17:52:38,455 INFO    SenderThread:1317 [sender.py:_start_run_threads():1136] run started: v53k76w9 with start time 1707414758.178795
+2024-02-08 17:52:38,459 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: check_version
+2024-02-08 17:52:38,459 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: check_version
+2024-02-08 17:52:38,542 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: run_start
+2024-02-08 17:52:38,571 DEBUG   HandlerThread:1317 [system_info.py:__init__():32] System info init
+2024-02-08 17:52:38,572 DEBUG   HandlerThread:1317 [system_info.py:__init__():47] System info init done
+2024-02-08 17:52:38,572 INFO    HandlerThread:1317 [system_monitor.py:start():194] Starting system monitor
+2024-02-08 17:52:38,572 INFO    SystemMonitor:1317 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-02-08 17:52:38,573 INFO    HandlerThread:1317 [system_monitor.py:probe():214] Collecting system info
+2024-02-08 17:52:38,574 INFO    SystemMonitor:1317 [interfaces.py:start():190] Started cpu monitoring
+2024-02-08 17:52:38,574 INFO    SystemMonitor:1317 [interfaces.py:start():190] Started disk monitoring
+2024-02-08 17:52:38,576 INFO    SystemMonitor:1317 [interfaces.py:start():190] Started gpu monitoring
+2024-02-08 17:52:38,578 INFO    SystemMonitor:1317 [interfaces.py:start():190] Started memory monitoring
+2024-02-08 17:52:38,579 INFO    SystemMonitor:1317 [interfaces.py:start():190] Started network monitoring
+2024-02-08 17:52:38,631 DEBUG   HandlerThread:1317 [system_info.py:probe():196] Probing system
+2024-02-08 17:52:38,633 DEBUG   HandlerThread:1317 [gitlib.py:_init_repo():56] git repository is invalid
+2024-02-08 17:52:38,633 DEBUG   HandlerThread:1317 [system_info.py:probe():244] Probing system done
+2024-02-08 17:52:38,633 DEBUG   HandlerThread:1317 [system_monitor.py:probe():223] {'os': 'Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35', 'python': '3.10.13', 'heartbeatAt': '2024-02-08T17:52:38.631590', 'startedAt': '2024-02-08T17:52:38.174980', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py', 'codePathLocal': None, 'host': 'default', 'username': 'sagemaker-user', 'executable': '/opt/conda/bin/python3', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 3096.191276041665, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2806.797, 'min': 0.0, 'max': 0.0}, {'current': 2305.192, 'min': 0.0, 'max': 0.0}, {'current': 2429.072, 'min': 0.0, 'max': 0.0}, {'current': 2448.527, 'min': 0.0, 'max': 0.0}, {'current': 2217.0, 'min': 0.0, 'max': 0.0}, {'current': 2733.2, 'min': 0.0, 'max': 0.0}, {'current': 2599.219, 'min': 0.0, 'max': 0.0}, {'current': 2830.092, 'min': 0.0, 'max': 0.0}, {'current': 2856.656, 'min': 0.0, 'max': 0.0}, {'current': 2766.239, 'min': 0.0, 'max': 0.0}, {'current': 2761.423, 'min': 0.0, 'max': 0.0}, {'current': 2600.369, 'min': 0.0, 'max': 0.0}, {'current': 2658.209, 'min': 0.0, 'max': 0.0}, {'current': 2747.075, 'min': 0.0, 'max': 0.0}, {'current': 3300.035, 'min': 0.0, 'max': 0.0}, {'current': 2742.13, 'min': 0.0, 'max': 0.0}, {'current': 2818.903, 'min': 0.0, 'max': 0.0}, {'current': 2743.213, 'min': 0.0, 'max': 0.0}, {'current': 2432.09, 'min': 0.0, 'max': 0.0}, {'current': 2731.02, 'min': 0.0, 'max': 0.0}, {'current': 2808.377, 'min': 0.0, 'max': 0.0}, {'current': 2777.618, 'min': 0.0, 'max': 0.0}, {'current': 2290.979, 'min': 0.0, 'max': 0.0}, {'current': 2230.543, 'min': 0.0, 'max': 0.0}, {'current': 2738.423, 'min': 0.0, 'max': 0.0}, {'current': 2903.95, 'min': 0.0, 'max': 0.0}, {'current': 2970.61, 'min': 0.0, 'max': 0.0}, {'current': 3299.839, 'min': 0.0, 'max': 0.0}, {'current': 2689.335, 'min': 0.0, 'max': 0.0}, {'current': 2791.925, 'min': 0.0, 'max': 0.0}, {'current': 2731.728, 'min': 0.0, 'max': 0.0}, {'current': 2813.357, 'min': 0.0, 'max': 0.0}, {'current': 2794.296, 'min': 0.0, 'max': 0.0}, {'current': 2747.123, 'min': 0.0, 'max': 0.0}, {'current': 2795.435, 'min': 0.0, 'max': 0.0}, {'current': 2767.017, 'min': 0.0, 'max': 0.0}, {'current': 2722.071, 'min': 0.0, 'max': 0.0}, {'current': 3298.527, 'min': 0.0, 'max': 0.0}, {'current': 2932.725, 'min': 0.0, 'max': 0.0}, {'current': 3292.093, 'min': 0.0, 'max': 0.0}, {'current': 3265.824, 'min': 0.0, 'max': 0.0}, {'current': 3256.045, 'min': 0.0, 'max': 0.0}, {'current': 3256.429, 'min': 0.0, 'max': 0.0}, {'current': 3259.575, 'min': 0.0, 'max': 0.0}, {'current': 2700.636, 'min': 0.0, 'max': 0.0}, {'current': 3234.186, 'min': 0.0, 'max': 0.0}, {'current': 3206.966, 'min': 0.0, 'max': 0.0}, {'current': 3299.085, 'min': 0.0, 'max': 0.0}, {'current': 3282.893, 'min': 0.0, 'max': 0.0}, {'current': 3279.04, 'min': 0.0, 'max': 0.0}, {'current': 3278.154, 'min': 0.0, 'max': 0.0}, {'current': 3283.989, 'min': 0.0, 'max': 0.0}, {'current': 2562.18, 'min': 0.0, 'max': 0.0}, {'current': 2954.006, 'min': 0.0, 'max': 0.0}, {'current': 2762.278, 'min': 0.0, 'max': 0.0}, {'current': 3275.22, 'min': 0.0, 'max': 0.0}, {'current': 3300.85, 'min': 0.0, 'max': 0.0}, {'current': 3291.939, 'min': 0.0, 'max': 0.0}, {'current': 2973.521, 'min': 0.0, 'max': 0.0}, {'current': 2966.002, 'min': 0.0, 'max': 0.0}, {'current': 2966.843, 'min': 0.0, 'max': 0.0}, {'current': 2645.143, 'min': 0.0, 'max': 0.0}, {'current': 3046.118, 'min': 0.0, 'max': 0.0}, {'current': 3006.852, 'min': 0.0, 'max': 0.0}, {'current': 3296.715, 'min': 0.0, 'max': 0.0}, {'current': 2922.754, 'min': 0.0, 'max': 0.0}, {'current': 2906.522, 'min': 0.0, 'max': 0.0}, {'current': 3028.907, 'min': 0.0, 'max': 0.0}, {'current': 2966.081, 'min': 0.0, 'max': 0.0}, {'current': 2917.105, 'min': 0.0, 'max': 0.0}, {'current': 3299.43, 'min': 0.0, 'max': 0.0}, {'current': 3300.481, 'min': 0.0, 'max': 0.0}, {'current': 3270.344, 'min': 0.0, 'max': 0.0}, {'current': 2930.864, 'min': 0.0, 'max': 0.0}, {'current': 2879.041, 'min': 0.0, 'max': 0.0}, {'current': 2902.742, 'min': 0.0, 'max': 0.0}, {'current': 3300.401, 'min': 0.0, 'max': 0.0}, {'current': 2686.543, 'min': 0.0, 'max': 0.0}, {'current': 3222.046, 'min': 0.0, 'max': 0.0}, {'current': 3298.97, 'min': 0.0, 'max': 0.0}, {'current': 3298.666, 'min': 0.0, 'max': 0.0}, {'current': 2754.074, 'min': 0.0, 'max': 0.0}, {'current': 3299.533, 'min': 0.0, 'max': 0.0}, {'current': 2812.149, 'min': 0.0, 'max': 0.0}, {'current': 3300.31, 'min': 0.0, 'max': 0.0}, {'current': 3300.208, 'min': 0.0, 'max': 0.0}, {'current': 2779.101, 'min': 0.0, 'max': 0.0}, {'current': 3300.477, 'min': 0.0, 'max': 0.0}, {'current': 2825.936, 'min': 0.0, 'max': 0.0}, {'current': 2204.979, 'min': 0.0, 'max': 0.0}, {'current': 2851.77, 'min': 0.0, 'max': 0.0}, {'current': 2797.024, 'min': 0.0, 'max': 0.0}, {'current': 2325.643, 'min': 0.0, 'max': 0.0}, {'current': 2850.865, 'min': 0.0, 'max': 0.0}, {'current': 2919.634, 'min': 0.0, 'max': 0.0}, {'current': 2910.972, 'min': 0.0, 'max': 0.0}, {'current': 2523.164, 'min': 0.0, 'max': 0.0}, {'current': 2297.34, 'min': 0.0, 'max': 0.0}, {'current': 2193.979, 'min': 0.0, 'max': 0.0}, {'current': 2128.798, 'min': 0.0, 'max': 0.0}, {'current': 1907.218, 'min': 0.0, 'max': 0.0}, {'current': 2921.246, 'min': 0.0, 'max': 0.0}, {'current': 2408.454, 'min': 0.0, 'max': 0.0}, {'current': 2296.906, 'min': 0.0, 'max': 0.0}, {'current': 2877.315, 'min': 0.0, 'max': 0.0}, {'current': 2985.576, 'min': 0.0, 'max': 0.0}, {'current': 2977.194, 'min': 0.0, 'max': 0.0}, {'current': 2982.705, 'min': 0.0, 'max': 0.0}, {'current': 2367.542, 'min': 0.0, 'max': 0.0}, {'current': 2232.475, 'min': 0.0, 'max': 0.0}, {'current': 2720.158, 'min': 0.0, 'max': 0.0}, {'current': 2260.753, 'min': 0.0, 'max': 0.0}, {'current': 2215.697, 'min': 0.0, 'max': 0.0}, {'current': 2278.892, 'min': 0.0, 'max': 0.0}, {'current': 2009.932, 'min': 0.0, 'max': 0.0}, {'current': 2813.45, 'min': 0.0, 'max': 0.0}, {'current': 2248.538, 'min': 0.0, 'max': 0.0}, {'current': 2789.291, 'min': 0.0, 'max': 0.0}, {'current': 2481.076, 'min': 0.0, 'max': 0.0}, {'current': 2033.475, 'min': 0.0, 'max': 0.0}, {'current': 2214.296, 'min': 0.0, 'max': 0.0}, {'current': 2762.868, 'min': 0.0, 'max': 0.0}, {'current': 2273.931, 'min': 0.0, 'max': 0.0}, {'current': 2891.192, 'min': 0.0, 'max': 0.0}, {'current': 2217.993, 'min': 0.0, 'max': 0.0}, {'current': 2306.666, 'min': 0.0, 'max': 0.0}, {'current': 2372.976, 'min': 0.0, 'max': 0.0}, {'current': 2322.672, 'min': 0.0, 'max': 0.0}, {'current': 2325.945, 'min': 0.0, 'max': 0.0}, {'current': 2332.493, 'min': 0.0, 'max': 0.0}, {'current': 2202.398, 'min': 0.0, 'max': 0.0}, {'current': 2130.875, 'min': 0.0, 'max': 0.0}, {'current': 2034.318, 'min': 0.0, 'max': 0.0}, {'current': 2539.829, 'min': 0.0, 'max': 0.0}, {'current': 2088.35, 'min': 0.0, 'max': 0.0}, {'current': 2427.524, 'min': 0.0, 'max': 0.0}, {'current': 2432.02, 'min': 0.0, 'max': 0.0}, {'current': 2521.716, 'min': 0.0, 'max': 0.0}, {'current': 3047.178, 'min': 0.0, 'max': 0.0}, {'current': 2452.92, 'min': 0.0, 'max': 0.0}, {'current': 2398.052, 'min': 0.0, 'max': 0.0}, {'current': 2930.232, 'min': 0.0, 'max': 0.0}, {'current': 2915.194, 'min': 0.0, 'max': 0.0}, {'current': 3050.935, 'min': 0.0, 'max': 0.0}, {'current': 2985.592, 'min': 0.0, 'max': 0.0}, {'current': 2999.519, 'min': 0.0, 'max': 0.0}, {'current': 2954.304, 'min': 0.0, 'max': 0.0}, {'current': 3253.761, 'min': 0.0, 'max': 0.0}, {'current': 2547.987, 'min': 0.0, 'max': 0.0}, {'current': 2791.034, 'min': 0.0, 'max': 0.0}, {'current': 2669.218, 'min': 0.0, 'max': 0.0}, {'current': 3304.846, 'min': 0.0, 'max': 0.0}, {'current': 3017.308, 'min': 0.0, 'max': 0.0}, {'current': 3299.861, 'min': 0.0, 'max': 0.0}, {'current': 2977.232, 'min': 0.0, 'max': 0.0}, {'current': 2939.823, 'min': 0.0, 'max': 0.0}, {'current': 3300.543, 'min': 0.0, 'max': 0.0}, {'current': 3014.24, 'min': 0.0, 'max': 0.0}, {'current': 3299.908, 'min': 0.0, 'max': 0.0}, {'current': 3014.885, 'min': 0.0, 'max': 0.0}, {'current': 3297.521, 'min': 0.0, 'max': 0.0}, {'current': 3296.848, 'min': 0.0, 'max': 0.0}, {'current': 3297.858, 'min': 0.0, 'max': 0.0}, {'current': 3296.813, 'min': 0.0, 'max': 0.0}, {'current': 2998.973, 'min': 0.0, 'max': 0.0}, {'current': 3299.759, 'min': 0.0, 'max': 0.0}, {'current': 3026.427, 'min': 0.0, 'max': 0.0}, {'current': 3300.35, 'min': 0.0, 'max': 0.0}, {'current': 2507.162, 'min': 0.0, 'max': 0.0}, {'current': 3250.875, 'min': 0.0, 'max': 0.0}, {'current': 3299.582, 'min': 0.0, 'max': 0.0}, {'current': 3299.791, 'min': 0.0, 'max': 0.0}, {'current': 2876.895, 'min': 0.0, 'max': 0.0}, {'current': 3300.637, 'min': 0.0, 'max': 0.0}, {'current': 3299.935, 'min': 0.0, 'max': 0.0}, {'current': 3299.409, 'min': 0.0, 'max': 0.0}, {'current': 3299.545, 'min': 0.0, 'max': 0.0}, {'current': 2845.582, 'min': 0.0, 'max': 0.0}, {'current': 3298.789, 'min': 0.0, 'max': 0.0}, {'current': 3212.048, 'min': 0.0, 'max': 0.0}, {'current': 2598.735, 'min': 0.0, 'max': 0.0}, {'current': 3299.632, 'min': 0.0, 'max': 0.0}, {'current': 3299.179, 'min': 0.0, 'max': 0.0}, {'current': 3298.805, 'min': 0.0, 'max': 0.0}, {'current': 3296.982, 'min': 0.0, 'max': 0.0}, {'current': 2498.549, 'min': 0.0, 'max': 0.0}, {'current': 3296.222, 'min': 0.0, 'max': 0.0}, {'current': 3297.448, 'min': 0.0, 'max': 0.0}, {'current': 2830.786, 'min': 0.0, 'max': 0.0}, {'current': 3299.116, 'min': 0.0, 'max': 0.0}, {'current': 3299.39, 'min': 0.0, 'max': 0.0}, {'current': 3299.373, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 32.0, 'used': 0.012481689453125}}, 'gpu': 'NVIDIA A10G', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}], 'memory': {'total': 747.9597625732422}}
+2024-02-08 17:52:38,634 INFO    HandlerThread:1317 [system_monitor.py:probe():224] Finished collecting system info
+2024-02-08 17:52:38,634 INFO    HandlerThread:1317 [system_monitor.py:probe():227] Publishing system info
+2024-02-08 17:52:38,634 DEBUG   HandlerThread:1317 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2024-02-08 17:52:38,634 DEBUG   HandlerThread:1317 [system_info.py:_save_pip():68] Saving pip packages done
+2024-02-08 17:52:38,634 DEBUG   HandlerThread:1317 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2024-02-08 17:52:39,456 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/conda-environment.yaml
+2024-02-08 17:52:39,457 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/requirements.txt
+2024-02-08 17:52:52,948 DEBUG   HandlerThread:1317 [system_info.py:_save_conda():87] Saving conda packages done
+2024-02-08 17:52:52,950 INFO    HandlerThread:1317 [system_monitor.py:probe():229] Finished publishing system info
+2024-02-08 17:52:52,954 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 17:52:52,954 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 17:52:52,954 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 17:52:52,954 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 17:52:52,955 DEBUG   SenderThread:1317 [sender.py:send():382] send: files
+2024-02-08 17:52:52,955 INFO    SenderThread:1317 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now
+2024-02-08 17:52:52,961 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-08 17:52:52,962 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: stop_status
+2024-02-08 17:52:52,964 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 17:52:53,118 DEBUG   SenderThread:1317 [sender.py:send():382] send: telemetry
+2024-02-08 17:52:53,118 DEBUG   SenderThread:1317 [sender.py:send():382] send: config
+2024-02-08 17:52:53,118 DEBUG   SenderThread:1317 [sender.py:send():382] send: metric
+2024-02-08 17:52:53,118 DEBUG   SenderThread:1317 [sender.py:send():382] send: telemetry
+2024-02-08 17:52:53,119 DEBUG   SenderThread:1317 [sender.py:send():382] send: metric
+2024-02-08 17:52:53,119 WARNING SenderThread:1317 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen)
+2024-02-08 17:52:53,356 INFO    wandb-upload_0:1317 [upload_job.py:push():131] Uploaded file /tmp/tmpftpllcuxwandb/1bgc597r-wandb-metadata.json
+2024-02-08 17:52:53,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/conda-environment.yaml
+2024-02-08 17:52:53,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/wandb-metadata.json
+2024-02-08 17:52:53,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/output.log
+2024-02-08 17:52:53,833 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 17:52:55,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/output.log
+2024-02-08 17:52:55,914 DEBUG   SenderThread:1317 [sender.py:send():382] send: exit
+2024-02-08 17:52:55,914 INFO    SenderThread:1317 [sender.py:send_exit():589] handling exit code: 1
+2024-02-08 17:52:55,914 INFO    SenderThread:1317 [sender.py:send_exit():591] handling runtime: 17
+2024-02-08 17:52:55,915 INFO    SenderThread:1317 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 17:52:55,915 INFO    SenderThread:1317 [sender.py:send_exit():597] send defer
+2024-02-08 17:52:55,915 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,915 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 0
+2024-02-08 17:52:55,916 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,916 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-02-08 17:52:55,916 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 1
+2024-02-08 17:52:55,916 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,916 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 1
+2024-02-08 17:52:55,916 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,916 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-02-08 17:52:55,916 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 2
+2024-02-08 17:52:55,916 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,916 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 2
+2024-02-08 17:52:55,916 INFO    HandlerThread:1317 [system_monitor.py:finish():203] Stopping system monitor
+2024-02-08 17:52:55,917 INFO    HandlerThread:1317 [interfaces.py:finish():202] Joined cpu monitor
+2024-02-08 17:52:55,917 INFO    HandlerThread:1317 [interfaces.py:finish():202] Joined disk monitor
+2024-02-08 17:52:55,918 DEBUG   SystemMonitor:1317 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-02-08 17:52:55,918 DEBUG   SystemMonitor:1317 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-02-08 17:52:55,918 DEBUG   SystemMonitor:1317 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-02-08 17:52:55,956 INFO    HandlerThread:1317 [interfaces.py:finish():202] Joined gpu monitor
+2024-02-08 17:52:55,956 INFO    HandlerThread:1317 [interfaces.py:finish():202] Joined memory monitor
+2024-02-08 17:52:55,956 INFO    HandlerThread:1317 [interfaces.py:finish():202] Joined network monitor
+2024-02-08 17:52:55,957 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,957 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-02-08 17:52:55,957 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 3
+2024-02-08 17:52:55,957 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,958 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 3
+2024-02-08 17:52:55,958 DEBUG   SenderThread:1317 [sender.py:send():382] send: stats
+2024-02-08 17:52:55,959 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,959 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-02-08 17:52:55,959 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 4
+2024-02-08 17:52:55,959 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,959 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 4
+2024-02-08 17:52:55,959 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,959 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-02-08 17:52:55,959 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 5
+2024-02-08 17:52:55,959 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,959 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 5
+2024-02-08 17:52:55,960 DEBUG   SenderThread:1317 [sender.py:send():382] send: summary
+2024-02-08 17:52:55,961 INFO    SenderThread:1317 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 17:52:55,961 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,961 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-02-08 17:52:55,961 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 6
+2024-02-08 17:52:55,961 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:55,961 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 6
+2024-02-08 17:52:55,961 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:55,961 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-02-08 17:52:55,966 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 17:52:56,102 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 7
+2024-02-08 17:52:56,102 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:56,102 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 7
+2024-02-08 17:52:56,103 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:56,103 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-02-08 17:52:56,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/config.yaml
+2024-02-08 17:52:56,459 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/wandb-summary.json
+2024-02-08 17:52:56,914 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 17:52:57,129 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 8
+2024-02-08 17:52:57,129 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 17:52:57,130 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:57,130 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 8
+2024-02-08 17:52:57,130 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:57,130 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-02-08 17:52:57,130 INFO    SenderThread:1317 [job_builder.py:build():298] Attempting to build job artifact
+2024-02-08 17:52:57,131 INFO    SenderThread:1317 [job_builder.py:_get_source_type():439] no source found
+2024-02-08 17:52:57,131 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 9
+2024-02-08 17:52:57,131 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:57,131 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 9
+2024-02-08 17:52:57,132 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:57,132 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-02-08 17:52:57,132 INFO    SenderThread:1317 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-02-08 17:52:57,460 INFO    Thread-12 :1317 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/output.log
+2024-02-08 17:52:57,460 INFO    SenderThread:1317 [dir_watcher.py:finish():388] scan: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files
+2024-02-08 17:52:57,460 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/config.yaml config.yaml
+2024-02-08 17:52:57,460 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/requirements.txt requirements.txt
+2024-02-08 17:52:57,460 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/conda-environment.yaml conda-environment.yaml
+2024-02-08 17:52:57,461 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/wandb-metadata.json wandb-metadata.json
+2024-02-08 17:52:57,461 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/output.log output.log
+2024-02-08 17:52:57,463 INFO    SenderThread:1317 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/wandb-summary.json wandb-summary.json
+2024-02-08 17:52:57,464 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 10
+2024-02-08 17:52:57,467 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:57,467 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 10
+2024-02-08 17:52:57,468 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:57,468 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-02-08 17:52:57,468 INFO    SenderThread:1317 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 17:52:57,674 INFO    wandb-upload_0:1317 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/requirements.txt
+2024-02-08 17:52:57,753 INFO    wandb-upload_1:1317 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/config.yaml
+2024-02-08 17:52:57,791 INFO    wandb-upload_3:1317 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/output.log
+2024-02-08 17:52:57,800 INFO    wandb-upload_2:1317 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/conda-environment.yaml
+2024-02-08 17:52:57,804 INFO    wandb-upload_4:1317 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/files/wandb-summary.json
+2024-02-08 17:52:57,915 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 17:52:57,915 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 17:52:58,004 INFO    Thread-11 (_thread_body):1317 [sender.py:transition_state():617] send defer: 11
+2024-02-08 17:52:58,004 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:58,004 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 11
+2024-02-08 17:52:58,005 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:58,005 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-02-08 17:52:58,005 INFO    SenderThread:1317 [file_pusher.py:join():181] waiting for file pusher
+2024-02-08 17:52:58,005 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 12
+2024-02-08 17:52:58,005 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:58,005 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 12
+2024-02-08 17:52:58,006 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:58,006 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-02-08 17:52:58,006 INFO    SenderThread:1317 [file_stream.py:finish():595] file stream finish called
+2024-02-08 17:52:58,071 INFO    SenderThread:1317 [file_stream.py:finish():599] file stream finish is done
+2024-02-08 17:52:58,071 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 13
+2024-02-08 17:52:58,071 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:58,071 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 13
+2024-02-08 17:52:58,071 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:58,071 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-02-08 17:52:58,071 INFO    SenderThread:1317 [sender.py:transition_state():617] send defer: 14
+2024-02-08 17:52:58,071 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 17:52:58,071 INFO    HandlerThread:1317 [handler.py:handle_request_defer():172] handle defer: 14
+2024-02-08 17:52:58,072 DEBUG   SenderThread:1317 [sender.py:send():382] send: final
+2024-02-08 17:52:58,072 DEBUG   SenderThread:1317 [sender.py:send():382] send: footer
+2024-02-08 17:52:58,072 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: defer
+2024-02-08 17:52:58,072 INFO    SenderThread:1317 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-02-08 17:52:58,072 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 17:52:58,072 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 17:52:58,073 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 17:52:58,073 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 17:52:58,073 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: server_info
+2024-02-08 17:52:58,073 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: server_info
+2024-02-08 17:52:58,075 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: get_summary
+2024-02-08 17:52:58,075 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: sampled_history
+2024-02-08 17:52:58,076 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 17:52:58,076 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: job_info
+2024-02-08 17:52:58,121 DEBUG   SenderThread:1317 [sender.py:send_request():409] send_request: job_info
+2024-02-08 17:52:58,122 INFO    MainThread:1317 [wandb_run.py:_footer_history_summary_info():3837] rendering history
+2024-02-08 17:52:58,122 INFO    MainThread:1317 [wandb_run.py:_footer_history_summary_info():3869] rendering summary
+2024-02-08 17:52:58,122 INFO    MainThread:1317 [wandb_run.py:_footer_sync_info():3796] logging synced files
+2024-02-08 17:52:58,122 DEBUG   HandlerThread:1317 [handler.py:handle_request():146] handle_request: shutdown
+2024-02-08 17:52:58,122 INFO    HandlerThread:1317 [handler.py:finish():866] shutting down handler
+2024-02-08 17:52:59,076 INFO    WriterThread:1317 [datastore.py:close():294] close: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/run-v53k76w9.wandb
+2024-02-08 17:52:59,122 INFO    SenderThread:1317 [sender.py:finish():1548] shutting down sender
+2024-02-08 17:52:59,122 INFO    SenderThread:1317 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 17:52:59,122 INFO    SenderThread:1317 [file_pusher.py:join():181] waiting for file pusher

wandb/run-20240208_175238-v53k76w9/logs/debug.log

@@ -0,0 +1,31 @@
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Configure stats pid to 1048
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/.config/wandb/settings
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/output-7b-26k-lora/wandb/settings
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-08 17:52:38,176 WARNING MainThread:1048 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/home/sagemaker-user/lora_finetuning_push_to_hub_save_local.py', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py'}
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:_log_setup():524] Logging user logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/logs/debug.log
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:_log_setup():525] Logging internal logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_175238-v53k76w9/logs/debug-internal.log
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:init():564] calling init triggers
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+config: {}
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:init():614] starting backend
+2024-02-08 17:52:38,176 INFO    MainThread:1048 [wandb_init.py:init():618] setting up manager
+2024-02-08 17:52:38,177 INFO    MainThread:1048 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-08 17:52:38,178 INFO    MainThread:1048 [wandb_init.py:init():624] backend started and connected
+2024-02-08 17:52:38,180 INFO    MainThread:1048 [wandb_init.py:init():716] updated telemetry
+2024-02-08 17:52:38,181 INFO    MainThread:1048 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-08 17:52:38,458 INFO    MainThread:1048 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-08 17:52:38,498 INFO    MainThread:1048 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.3 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-02-08 17:52:38,498 INFO    MainThread:1048 [wandb_init.py:init():800] starting run threads in backend
+2024-02-08 17:52:52,962 INFO    MainThread:1048 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-08 17:52:52,963 INFO    MainThread:1048 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-08 17:52:52,963 INFO    MainThread:1048 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-08 17:52:52,963 INFO    MainThread:1048 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-08 17:52:52,963 INFO    MainThread:1048 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-08 17:52:52,965 INFO    MainThread:1048 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 102400, 'max_position_embeddings': 4096, 'hidden_size': 4096, 'intermediate_size': 11008, 'num_hidden_layers': 30, 'num_attention_heads': 32, 'num_key_value_heads': 32, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_scaling': None, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 100000, 'pad_token_id': 0, 'eos_token_id': 100015, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'deepseek-ai/deepseek-coder-7b-instruct-v1.5', 'transformers_version': '4.31.0', 'attention_bias': False, 'model_type': 'llama', 'rope_theta': 10000.0, 'output_dir': 'output-7b-26k-lora', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'output-7b-26k-lora/runs/Feb08_17-51-59_default', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': 100, 'save_safetensors': False, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'ddp_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 4, 'past_index': -1, 'run_name': 'output-7b-26k-lora', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': True, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'ddp_broadcast_buffers': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'zzz99/output-7b-26k-lora', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'xpu_backend': 'None', 'cache_dir': 'None', 'model_max_length': 960, 'gradient_checkpointing_kwargs': "{'use_reentrant': False}", 'train_batch_size': 4, 'eval_batch_size': 1}
+2024-02-08 17:52:59,230 WARNING MsgRouterThr:1048 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240208_180020-3wdew33h/files/conda-environment.yaml

@@ -0,0 +1,662 @@
+name: base
+channels:
+  - file:///tmp/local_conda_forge_artifacts
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - _py-xgboost-mutex=2.0=gpu_0
+  - absl-py=2.0.0=pyhd8ed1ab_0
+  - accelerate=0.21.0=pyhd8ed1ab_0
+  - aiobotocore=2.7.0=pyhd8ed1ab_1
+  - aiohttp=3.9.1=py310h2372a71_0
+  - aioitertools=0.11.0=pyhd8ed1ab_0
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - aiosqlite=0.19.0=pyhd8ed1ab_0
+  - alsa-lib=1.2.10=hd590300_0
+  - altair=5.2.0=pyhd8ed1ab_0
+  - amazon-codewhisperer-jupyterlab-ext=2.0.1=pyhd8ed1ab_0
+  - amazon-sagemaker-jupyter-scheduler=3.0.5=pyhd8ed1ab_0
+  - ansi2html=1.8.0=py310hff52083_2
+  - ansiwrap=0.8.4=py_0
+  - antlr-python-runtime=4.9.3=pyhd8ed1ab_1
+  - anyio=3.7.1=pyhd8ed1ab_0
+  - aom=3.5.0=h27087fc_0
+  - appdirs=1.4.4=pyh9f0ad1d_0
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - astroid=3.0.1=py310hff52083_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - astunparse=1.6.3=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - async-timeout=4.0.3=pyhd8ed1ab_0
+  - atk-1.0=2.38.0=hd4edc92_1
+  - attr=2.5.1=h166bdaf_1
+  - attrs=23.1.0=pyh71513ae_1
+  - autogluon=0.8.2=pyhd8ed1ab_4
+  - autogluon.common=0.8.2=pyhd8ed1ab_4
+  - autogluon.core=0.8.2=light_py310h2d11d36_6
+  - autogluon.features=0.8.2=pyhd8ed1ab_3
+  - autogluon.multimodal=0.8.2=pyha770c72_5
+  - autogluon.tabular=0.8.2=pyha770c72_4
+  - autogluon.timeseries=0.8.2=pyhd8ed1ab_4
+  - autopep8=2.0.4=pyhd8ed1ab_0
+  - autovizwidget=0.21.0=pyh1a96a4e_0
+  - aws-c-auth=0.7.3=h28f7589_1
+  - aws-c-cal=0.6.1=hc309b26_1
+  - aws-c-common=0.9.0=hd590300_0
+  - aws-c-compression=0.2.17=h4d4d85c_2
+  - aws-c-event-stream=0.3.1=h2e3709c_4
+  - aws-c-http=0.7.11=h00aa349_4
+  - aws-c-io=0.13.32=he9a53bd_1
+  - aws-c-mqtt=0.9.3=hb447be9_1
+  - aws-c-s3=0.3.14=hf3aad02_1
+  - aws-c-sdkutils=0.1.12=h4d4d85c_1
+  - aws-checksums=0.1.17=h4d4d85c_1
+  - aws-crt-cpp=0.21.0=hb942446_5
+  - aws-embedded-metrics=3.2.0=pyhd8ed1ab_0
+  - aws-glue-sessions=1.0.4=pyhd8ed1ab_0
+  - aws-sdk-cpp=1.10.57=h85b1a90_19
+  - babel=2.13.1=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.2=pyha770c72_0
+  - binaryornot=0.4.4=py_1
+  - black=23.11.0=py310hff52083_0
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - blinker=1.7.0=pyhd8ed1ab_0
+  - blosc=1.21.5=h0f2a231_0
+  - boltons=23.0.0=pyhd8ed1ab_0
+  - boto3=1.28.64=pyhd8ed1ab_0
+  - botocore=1.31.64=pyhd8ed1ab_0
+  - brotli=1.0.9=h166bdaf_9
+  - brotli-bin=1.0.9=h166bdaf_9
+  - brotli-python=1.0.9=py310hd8f1fbe_9
+  - brunsli=0.1=h9c3ff4c_0
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.23.0=hd590300_0
+  - c-blosc2=2.11.3=hb4ffafa_0
+  - ca-certificates=2023.11.17=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cachetools=5.3.2=pyhd8ed1ab_0
+  - cairo=1.18.0=h3faef2a_0
+  - catalogue=2.0.10=py310hff52083_0
+  - catboost=1.2.2=py310hff52083_0
+  - certifi=2023.11.17=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - chardet=5.2.0=py310hff52083_1
+  - charls=2.4.2=h59595ed_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - click=8.1.7=unix_pyh707e725_0
+  - cloudpathlib=0.16.0=pyhd8ed1ab_0
+  - cloudpickle=2.2.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - conda=23.11.0=py310hff52083_1
+  - conda-libmamba-solver=23.11.1=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - confection=0.1.4=py310h17c5347_0
+  - contextlib2=21.6.0=pyhd8ed1ab_0
+  - contourpy=1.2.0=py310hd41b1e2_0
+  - cookiecutter=2.5.0=pyhca7485f_0
+  - croniter=1.4.1=pyhd8ed1ab_0
+  - cryptography=41.0.7=py310hb8475ec_1
+  - cuda-version=11.8=h70ddcb2_2
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudnn=8.8.0.121=hcdd5f01_4
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - cymem=2.0.8=py310hc6cd4ac_1
+  - cyrus-sasl=2.1.27=h54b06d7_7
+  - cython-blis=0.7.10=py310h1f7b6fc_2
+  - cytoolz=0.12.2=py310h2372a71_1
+  - dash=2.14.2=pyhd8ed1ab_0
+  - dask-core=2023.12.0=pyhd8ed1ab_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - dataclasses-json=0.6.3=pyhd8ed1ab_0
+  - datasets=2.14.7=pyhd8ed1ab_0
+  - dav1d=1.2.1=hd590300_0
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.8.0=py310hc6cd4ac_1
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - deepmerge=1.1.0=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - dill=0.3.7=pyhd8ed1ab_0
+  - distributed=2023.12.0=pyhd8ed1ab_0
+  - distro=1.8.0=pyhd8ed1ab_0
+  - docker-pycreds=0.4.0=py_0
+  - docstring-to-markdown=0.13=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - et_xmlfile=1.1.0=pyhd8ed1ab_0
+  - evaluate=0.4.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.5.0=hcb278e6_1
+  - faiss=1.7.4=py310h9ed8947_0_cpu
+  - faiss-cpu=1.7.4=h718b53a_0
+  - fastai=2.7.13=pyhd8ed1ab_0
+  - fastcore=1.5.29=pyhd8ed1ab_0
+  - fastdownload=0.0.7=pyhd8ed1ab_0
+  - fastprogress=1.0.3=pyhd8ed1ab_0
+  - filelock=3.13.1=pyhd8ed1ab_0
+  - flake8=6.1.0=pyhd8ed1ab_0
+  - flask=3.0.0=pyhd8ed1ab_0
+  - flatbuffers=23.5.26=h59595ed_1
+  - fmt=10.1.1=h00ab1b0_1
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_1
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.46.0=py310h2372a71_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - frozenlist=1.4.0=py310h2372a71_1
+  - fsspec=2023.6.0=pyh1a96a4e_0
+  - future=0.18.3=pyhd8ed1ab_0
+  - gast=0.4.0=pyh9f0ad1d_0
+  - gdk-pixbuf=2.42.10=h6b639ba_2
+  - gdown=4.7.1=pyhd8ed1ab_0
+  - gettext=0.21.1=h27087fc_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h0b41bf4_3
+  - gitdb=4.0.11=pyhd8ed1ab_0
+  - gitpython=3.1.40=pyhd8ed1ab_0
+  - glib=2.78.3=hfc55251_0
+  - glib-tools=2.78.3=hfc55251_0
+  - glog=0.6.0=h6f12383_0
+  - gluonts=0.13.7=pyhd8ed1ab_0
+  - gmp=6.3.0=h59595ed_0
+  - gmpy2=2.1.2=py310h3ec546c_1
+  - google-api-core=2.15.0=pyhd8ed1ab_0
+  - google-auth=2.25.2=pyhca7485f_0
+  - google-auth-oauthlib=1.0.0=pyhd8ed1ab_1
+  - google-pasta=0.2.0=pyh8c360ce_0
+  - googleapis-common-protos=1.62.0=pyhd8ed1ab_0
+  - graphite2=1.3.13=h58526e2_1001
+  - graphviz=8.1.0=h28d9a01_0
+  - grpcio=1.54.3=py310heca2aa9_0
+  - gst-plugins-base=1.22.7=h8e1006c_1
+  - gstreamer=1.22.7=h98fc4e7_1
+  - gtk2=2.24.33=h90689f9_2
+  - gts=0.7.6=h977cf35_4
+  - h5py=3.10.0=nompi_py310ha2ad45a_100
+  - harfbuzz=8.3.0=h3d44ed6_0
+  - hdf5=1.14.2=nompi_h4f84152_100
+  - hdijupyterutils=0.21.0=pyh1a96a4e_0
+  - huggingface_hub=0.19.4=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - imagecodecs=2023.8.12=py310hc929067_0
+  - imageio=2.31.5=pyh8c1a49c_0
+  - importlib-metadata=6.10.0=pyha770c72_0
+  - importlib_metadata=6.10.0=hd8ed1ab_0
+  - importlib_resources=6.1.1=pyhd8ed1ab_0
+  - iniconfig=2.0.0=pyhd8ed1ab_0
+  - ipykernel=6.26.0=pyhf8b6a83_0
+  - ipython=8.18.1=pyh707e725_3
+  - ipywidgets=8.1.1=pyhd8ed1ab_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - isort=5.12.0=pyhd8ed1ab_1
+  - itsdangerous=2.1.2=pyhd8ed1ab_0
+  - jax=0.4.20=pyhd8ed1ab_0
+  - jaxlib=0.4.14=cpu_py310h67d73b5_1
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.3.2=pyhd8ed1ab_0
+  - json5=0.9.14=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpath-ng=1.6.0=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.17.3=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.17.3=pyhd8ed1ab_0
+  - jupyter=1.0.0=pyhd8ed1ab_10
+  - jupyter-ai=2.6.0=pyhd8ed1ab_2
+  - jupyter-ai-magics=2.6.0=pyhd8ed1ab_1
+  - jupyter-dash=0.4.2=pyhd8ed1ab_1
+  - jupyter-lsp=2.2.1=pyhd8ed1ab_0
+  - jupyter-server-mathjax=0.2.6=pyh5bfe37b_1
+  - jupyter-server-proxy=4.1.0=pyhd8ed1ab_0
+  - jupyter_client=8.6.0=pyhd8ed1ab_0
+  - jupyter_console=6.6.3=pyhd8ed1ab_0
+  - jupyter_core=5.5.0=py310hff52083_0
+  - jupyter_events=0.6.3=pyhd8ed1ab_1
+  - jupyter_scheduler=2.4.0=pyha770c72_1
+  - jupyter_server=2.10.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0
+  - jupyterlab=4.0.9=pyhd8ed1ab_0
+  - jupyterlab-git=0.50.0=pyhd8ed1ab_0
+  - jupyterlab-lsp=5.0.1=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0
+  - jupyterlab_server=2.24.0=pyhd8ed1ab_0
+  - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0
+  - jxrlib=1.1=h7f98852_2
+  - keras=2.12.0=pyhd8ed1ab_0
+  - keras-preprocessing=1.1.2=pyhd8ed1ab_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py310hd41b1e2_1
+  - lame=3.100=h166bdaf_1003
+  - langchain=0.0.318=pyhd8ed1ab_0
+  - langcodes=3.3.0=pyhd8ed1ab_0
+  - langsmith=0.0.69=pyhd8ed1ab_0
+  - lcms2=2.15=haa2dc70_1
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20230125.3=cxx17_h59595ed_0
+  - libaec=1.1.2=h59595ed_1
+  - libarchive=3.7.2=h039dbb9_0
+  - libarrow=12.0.1=hb87d912_8_cpu
+  - libavif=0.11.1=h8182462_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libbrotlicommon=1.0.9=h166bdaf_9
+  - libbrotlidec=1.0.9=h166bdaf_9
+  - libbrotlienc=1.0.9=h166bdaf_9
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=16_linux64_mkl
+  - libclang=15.0.7=default_hb11cfb5_4
+  - libclang13=15.0.7=default_ha2b6cf4_4
+  - libcrc32c=1.1.2=h9c3ff4c_0
+  - libcups=2.3.3=h4637d8d_4
+  - libcurl=8.4.0=hca28451_0
+  - libdeflate=1.18=h0b41bf4_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.5.0=hcb278e6_1
+  - libfaiss=1.7.4=hf47d654_0_cpu
+  - libfaiss-avx2=1.7.4=h1234567_0_cpu
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgcrypt=1.10.3=hd590300_0
+  - libgd=2.3.3=h74d50f4_7
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libglib=2.78.3=h783c2da_0
+  - libgoogle-cloud=2.12.0=hac9eb74_1
+  - libgpg-error=1.47=h71f35ed_0
+  - libgrpc=1.54.3=hb20ce57_0
+  - libhwloc=2.9.3=default_h554bfaf_1009
+  - libiconv=1.17=hd590300_1
+  - libjpeg-turbo=2.1.5.1=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libllvm14=14.0.6=hcd5def8_4
+  - libllvm15=15.0.7=h5cf9203_3
+  - libmagma=2.7.1=hc72dce7_6
+  - libmagma_sparse=2.7.1=h8354cda_6
+  - libmamba=1.5.3=had39da4_2
+  - libmambapy=1.5.3=py310h39ff949_2
+  - libnghttp2=1.58.0=h47da74e_0
+  - libnsl=2.0.1=hd590300_0
+  - libntlm=1.4=h7f98852_1002
+  - libnuma=2.0.16=h0b41bf4_1
+  - libogg=1.3.4=h7f98852_1
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.39=h753d276_0
+  - libpq=15.4=hfc447b1_2
+  - libprotobuf=3.21.12=hfc55251_2
+  - librsvg=2.56.3=h98fae49_0
+  - libsecret=0.18.8=h329b89f_2
+  - libsentencepiece=0.1.99=h180e1df_0
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libsystemd0=255=h3516f8a_0
+  - libthrift=0.18.1=h8fd135c_2
+  - libtiff=4.5.1=h8b53f26_1
+  - libtool=2.4.7=h27087fc_0
+  - libutf8proc=2.8.0=h166bdaf_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libuv=1.46.0=hd590300_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp=1.3.1=hbf2b3c1_0
+  - libwebp-base=1.3.1=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxgboost=1.7.6=cuda118_hd3b444d_6
+  - libxkbcommon=1.6.0=h5d7e998_0
+  - libxml2=2.11.6=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - libzopfli=1.0.3=h9c3ff4c_0
+  - lightgbm=3.3.5=py310heca2aa9_0
+  - lightning-utilities=0.10.0=pyhd8ed1ab_0
+  - llvm-openmp=17.0.6=h4dfa4b3_0
+  - llvmlite=0.41.1=py310h1b8f574_0
+  - locket=1.0.0=pyhd8ed1ab_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - magma=2.7.1=ha770c72_6
+  - markdown=3.5.1=pyhd8ed1ab_0
+  - markdown-it-py=3.0.0=pyhd8ed1ab_0
+  - markupsafe=2.1.3=py310h2372a71_1
+  - marshmallow=3.20.1=pyhd8ed1ab_0
+  - matplotlib=3.8.2=py310hff52083_0
+  - matplotlib-base=3.8.2=py310h62c0568_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mccabe=0.7.0=pyhd8ed1ab_0
+  - mdurl=0.1.0=pyhd8ed1ab_0
+  - menuinst=2.0.1=py310hff52083_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2022.2.1=h84fe81f_16997
+  - ml_dtypes=0.3.1=py310hcc13569_2
+  - mlforecast=0.7.3=pyhd8ed1ab_0
+  - mock=5.1.0=pyhd8ed1ab_0
+  - model-index=0.1.11=pyhd8ed1ab_0
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h9458935_0
+  - mpg123=1.32.3=h59595ed_0
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - msgpack-python=1.0.7=py310hd41b1e2_0
+  - multidict=6.0.4=py310h2372a71_1
+  - multiprocess=0.70.15=py310h2372a71_1
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.10=py310hc6cd4ac_1
+  - mypy_extensions=1.0.0=pyha770c72_0
+  - mysql-common=8.0.33=hf1915f5_6
+  - mysql-libs=8.0.33=hca2cd23_6
+  - nbclient=0.8.0=pyhd8ed1ab_0
+  - nbconvert=7.12.0=pyhd8ed1ab_0
+  - nbconvert-core=7.12.0=pyhd8ed1ab_0
+  - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0
+  - nbdime=4.0.1=pyhd8ed1ab_0
+  - nbformat=5.9.2=pyhd8ed1ab_0
+  - nccl=2.19.4.1=h6103f9b_0
+  - ncurses=6.4=h59595ed_2
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - networkx=3.2.1=pyhd8ed1ab_0
+  - nlpaug=1.1.11=pyhd8ed1ab_1
+  - nltk=3.8.1=pyhd8ed1ab_0
+  - nodejs=18.18.2=hb753e55_1
+  - nose=1.3.7=py_1006
+  - notebook=7.0.6=pyhd8ed1ab_0
+  - notebook-shim=0.2.3=pyhd8ed1ab_0
+  - nptyping=2.4.1=pyhd8ed1ab_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.95=h1d7d5a4_0
+  - numba=0.58.1=py310h7dc5dd1_0
+  - numexpr=2.8.7=mkl_py310h5e07f1c_4
+  - numpy=1.26.2=py310hb13e2d6_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - omegaconf=2.2.3=pyhd8ed1ab_0
+  - openai=0.28.1=pyhd8ed1ab_0
+  - openapi-schema-pydantic=1.2.4=pyhd8ed1ab_0
+  - openjpeg=2.5.0=hfec8fc6_2
+  - openmim=0.3.7=pyhd8ed1ab_0
+  - openpyxl=3.1.2=py310h2372a71_1
+  - openssl=3.1.4=hd590300_0
+  - opt-einsum=3.3.0=hd8ed1ab_2
+  - opt_einsum=3.3.0=pyhc1e730c_2
+  - orc=1.9.0=h2f23424_1
+  - ordered-set=4.1.0=pyhd8ed1ab_0
+  - overrides=7.4.0=pyhd8ed1ab_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pandas=2.1.4=py310hcc13569_0
+  - pandas-stubs=2.1.1.230928=pyhd8ed1ab_1
+  - pandoc=3.1.3=h32600fe_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - pango=1.50.14=ha41ecd1_2
+  - papermill=2.4.0=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - partd=1.4.1=pyhd8ed1ab_0
+  - pathos=0.3.1=pyhd8ed1ab_0
+  - pathspec=0.12.1=pyhd8ed1ab_0
+  - pathtools=0.1.2=py_1
+  - pathy=0.10.2=pyhd8ed1ab_0
+  - patsy=0.5.4=pyhd8ed1ab_0
+  - pcre2=10.42=hcad00b1_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=9.5.0=py310h582fbeb_1
+  - pip=23.3.1=pyhd8ed1ab_0
+  - pixman=0.42.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - plotly=5.18.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - ply=3.11=py_1
+  - pox=0.3.3=pyhd8ed1ab_0
+  - ppft=1.7.6.7=pyhd8ed1ab_0
+  - preshed=3.0.9=py310hc6cd4ac_1
+  - prometheus_client=0.19.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.41=pyha770c72_0
+  - prompt_toolkit=3.0.41=hd8ed1ab_0
+  - protobuf=4.21.12=py310heca2aa9_0
+  - psutil=5.9.5=py310h2372a71_1
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio-client=16.1=hb77b528_5
+  - pure-sasl=0.6.2=pyhd8ed1ab_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - py-xgboost=1.7.6=cuda118_py310h9840055_6
+  - py-xgboost-gpu=1.7.6=py310h9840055_6
+  - pyarrow=12.0.1=py310h0576679_8_cpu
+  - pyarrow-hotfix=0.6=pyhd8ed1ab_0
+  - pyasn1=0.5.1=pyhd8ed1ab_0
+  - pyasn1-modules=0.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycodestyle=2.11.1=pyhd8ed1ab_0
+  - pycosat=0.6.6=py310h2372a71_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydocstyle=6.3.0=pyhd8ed1ab_0
+  - pyflakes=3.1.0=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pyhive=0.7.0=pyhd8ed1ab_0
+  - pyjwt=2.8.0=pyhd8ed1ab_0
+  - pylint=3.0.3=pyhd8ed1ab_0
+  - pyopenssl=23.3.0=pyhd8ed1ab_0
+  - pyparsing=3.1.1=pyhd8ed1ab_0
+  - pyqt=5.15.9=py310h04931ad_5
+  - pyqt5-sip=12.12.2=py310hc6cd4ac_5
+  - pyrsistent=0.20.0=py310h2372a71_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - pyspnego=0.9.1=py310h2372a71_2
+  - pytesseract=0.3.10=pyhd8ed1ab_0
+  - pytest=7.4.3=pyhd8ed1ab_0
+  - pytest-subtests=0.11.0=pyhd8ed1ab_0
+  - python=3.10.13=hd12c33a_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.0=pyhd8ed1ab_0
+  - python-flatbuffers=23.5.26=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=pyh22cad53_0
+  - python-gssapi=1.8.3=py310h98feede_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python-lsp-jsonrpc=1.1.2=pyhd8ed1ab_0
+  - python-lsp-server=1.9.0=pyhd8ed1ab_0
+  - python-lsp-server-base=1.9.0=pyhd8ed1ab_0
+  - python-slugify=8.0.1=pyhd8ed1ab_2
+  - python-tzdata=2023.3=pyhd8ed1ab_0
+  - python-xxhash=3.4.1=py310h2372a71_0
+  - python_abi=3.10=4_cp310
+  - pytoolconfig=1.2.5=pyhd8ed1ab_0
+  - pytorch=2.0.0=cuda112py310he33e0d6_200
+  - pytorch-gpu=2.0.0=cuda112py310h9871d0b_200
+  - pytorch-lightning=2.0.9=pyhd8ed1ab_0
+  - pytorch-metric-learning=1.7.3=pyhd8ed1ab_0
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywavelets=1.4.1=py310h1f7b6fc_1
+  - pyyaml=6.0.1=py310h2372a71_1
+  - pyzmq=25.1.2=py310h795f18f_0
+  - qt-main=5.15.8=hc47bfe8_16
+  - qtconsole-base=5.5.1=pyha770c72_0
+  - qtpy=2.4.1=pyhd8ed1ab_0
+  - rdma-core=28.9=h59595ed_1
+  - re2=2023.03.02=h8c504da_0
+  - readline=8.2=h8228510_1
+  - regex=2023.10.3=py310h2372a71_0
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - requests-kerberos=0.14.0=pyhd8ed1ab_1
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - retrying=1.3.3=py_2
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rich=13.7.0=pyhd8ed1ab_0
+  - ripgrep=14.0.3=he8a937b_1
+  - rope=1.11.0=pyhd8ed1ab_1
+  - rsa=4.9=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py310h2372a71_0
+  - ruamel.yaml.clib=0.2.7=py310h2372a71_2
+  - s2n=1.3.49=h06160fa_0
+  - s3transfer=0.7.0=pyhd8ed1ab_1
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - safetensors=0.3.3=py310hcb5633a_1
+  - sagemaker-code-editor=1.83.1=ha1e3dc1_3
+  - sagemaker-headless-execution-driver=0.0.12=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-emr-extension=0.1.9=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension=0.2.0=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension-common=0.1.9=pyhd8ed1ab_0
+  - sagemaker-kernel-wrapper=0.0.2=pyhd8ed1ab_0
+  - sagemaker-python-sdk=2.198.0=pyhd8ed1ab_0
+  - sagemaker-studio-analytics-extension=0.0.21=pyhd8ed1ab_0
+  - sagemaker-studio-sparkmagic-lib=0.1.4=pyhd8ed1ab_0
+  - sasl=0.3.1=py310hd8f1fbe_2
+  - schema=0.7.5=pyhd8ed1ab_1
+  - scikit-image=0.19.3=py310h769672d_2
+  - scikit-learn=1.3.2=py310h1fdf081_2
+  - scipy=1.11.4=py310hb13e2d6_0
+  - send2trash=1.8.2=pyh41d4057_0
+  - sentencepiece=0.1.99=hff52083_0
+  - sentencepiece-python=0.1.99=py310hebdb9f0_0
+  - sentencepiece-spm=0.1.99=h180e1df_0
+  - sentry-sdk=1.38.0=pyhd8ed1ab_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setproctitle=1.3.3=py310h2372a71_0
+  - setuptools=68.2.2=pyhd8ed1ab_0
+  - shellingham=1.5.4=pyhd8ed1ab_0
+  - simpervisor=1.0.0=pyhd8ed1ab_0
+  - sip=6.7.12=py310hc6cd4ac_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sleef=3.5.1=h9b69904_2
+  - smart_open=5.2.1=pyhd8ed1ab_0
+  - smdebug-rulesconfig=1.0.1=pyhd3deb0d_1
+  - smmap=5.0.0=pyhd8ed1ab_0
+  - snappy=1.1.10=h9fff704_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - snowballstemmer=2.2.0=pyhd8ed1ab_0
+  - sortedcontainers=2.4.0=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - spacy=3.7.2=py310hcb52e73_0
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.5=pyhd8ed1ab_0
+  - sparkmagic=0.21.0=pyhd8ed1ab_0
+  - srsly=2.4.8=py310hc6cd4ac_1
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - statsforecast=1.4.0=pyhd8ed1ab_0
+  - statsmodels=0.14.0=py310h1f7b6fc_3
+  - supervisor=4.2.5=py310hff52083_2
+  - sympy=1.12=pypyh9d50eac_103
+  - tabulate=0.9.0=pyhd8ed1ab_1
+  - tbb=2021.11.0=h00ab1b0_0
+  - tblib=1.7.0=pyhd8ed1ab_0
+  - tenacity=8.2.3=pyhd8ed1ab_0
+  - tensorboard=2.12.3=pyhd8ed1ab_0
+  - tensorboard-data-server=0.7.0=py310h75e40e8_1
+  - tensorflow=2.12.1=cuda112py310h457873b_1
+  - tensorflow-base=2.12.1=cuda112py310h1d65785_1
+  - tensorflow-estimator=2.12.1=cuda112py310ha5e6de5_1
+  - termcolor=2.3.0=pyhd8ed1ab_0
+  - terminado=0.18.0=pyh0d859eb_0
+  - text-unidecode=1.3=pyhd8ed1ab_1
+  - textwrap3=0.9.2=py_0
+  - thinc=8.2.1=py310hcb52e73_0
+  - threadpoolctl=3.2.0=pyha21a80b_0
+  - thrift=0.19.0=py310hc6cd4ac_1
+  - thrift_sasl=0.4.3=pyhd8ed1ab_2
+  - tifffile=2023.12.9=pyhd8ed1ab_0
+  - tiktoken=0.5.2=py310h47bb294_0
+  - timm=0.9.12=pyhd8ed1ab_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tokenizers=0.13.3=py310he1f1126_0
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tomlkit=0.12.3=pyha770c72_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchmetrics=1.0.3=pyhd8ed1ab_0
+  - torchvision=0.15.2=cuda112py310h0801bf5_1
+  - tornado=6.3.3=py310h2372a71_1
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - traitlets=5.14.0=pyhd8ed1ab_0
+  - transformers=4.31.0=pyhd8ed1ab_0
+  - truststore=0.8.0=pyhd8ed1ab_0
+  - typer=0.9.0=pyhd8ed1ab_0
+  - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0
+  - types-pytz=2023.3.1.1=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - typing_inspect=0.9.0=pyhd8ed1ab_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - typish=1.9.3=pyhd8ed1ab_0
+  - tzdata=2023c=h71feb2d_0
+  - ucx=1.14.1=h64cca9d_5
+  - ujson=5.9.0=py310hc6cd4ac_0
+  - unicodedata2=15.1.0=py310h2372a71_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - urllib3=1.26.18=pyhd8ed1ab_0
+  - wandb=0.16.1=pyhd8ed1ab_0
+  - wasabi=1.1.2=py310hff52083_0
+  - wcwidth=0.2.12=pyhd8ed1ab_0
+  - weasel=0.3.4=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.7.0=pyhd8ed1ab_0
+  - werkzeug=3.0.1=pyhd8ed1ab_0
+  - whatthepatch=1.0.5=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - widgetsnbextension=4.0.9=pyhd8ed1ab_0
+  - window-ops=0.0.14=pyhd8ed1ab_0
+  - wrapt=1.16.0=py310h2372a71_0
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xgboost=1.7.6=cuda118_py310h9840055_6
+  - xkeyboard-config=2.40=hd590300_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.7=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xxhash=0.8.2=hd590300_0
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yaml-cpp=0.8.0=h59595ed_0
+  - yapf=0.40.1=pyhd8ed1ab_0
+  - yarl=1.9.3=py310h2372a71_0
+  - zeromq=4.3.5=h59595ed_0
+  - zfp=1.0.0=h59595ed_4
+  - zict=3.0.0=pyhd8ed1ab_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zlib-ng=2.0.7=h0b41bf4_0
+  - zstandard=0.22.0=py310h1275a96_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - annotated-types==0.6.0
+      - greenlet==3.0.1
+      - jupyter-scheduler==2.4.0
+      - krb5==0.5.1
+      - peft==0.8.2
+      - pydantic==2.5.1
+      - pydantic-core==2.14.3
+      - sqlalchemy==1.4.50
+prefix: /opt/conda

wandb/run-20240208_180020-3wdew33h/files/config.yaml

@@ -0,0 +1,616 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.13
+    cli_version: 0.16.1
+    framework: huggingface
+    huggingface_version: 4.31.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1707415220.764703
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      3:
+      - 7
+      - 23
+      4: 3.10.13
+      5: 0.16.1
+      6: 4.31.0
+      8:
+      - 5
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+vocab_size:
+  desc: null
+  value: 32256
+max_position_embeddings:
+  desc: null
+  value: 16384
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 11008
+num_hidden_layers:
+  desc: null
+  value: 32
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 32
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_scaling:
+  desc: null
+  value:
+    factor: 4.0
+    type: linear
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 32013
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 32021
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: deepseek-ai/deepseek-coder-6.7b-instruct
+transformers_version:
+  desc: null
+  value: 4.31.0
+attention_bias:
+  desc: null
+  value: false
+model_type:
+  desc: null
+  value: llama
+rope_theta:
+  desc: null
+  value: 100000
+output_dir:
+  desc: null
+  value: output-67b-26k-lora
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: epoch
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_device_eval_batch_size:
+  desc: null
+  value: 1
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 2.0e-05
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 2
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 10
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: output-67b-26k-lora/runs/Feb08_17-59-25_default
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_steps:
+  desc: null
+  value: 100
+save_total_limit:
+  desc: null
+  value: 100
+save_safetensors:
+  desc: null
+  value: false
+save_on_each_node:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: None
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: None
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: None
+tpu_num_cores:
+  desc: null
+  value: None
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: '[]'
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: None
+dataloader_num_workers:
+  desc: null
+  value: 4
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: output-67b-26k-lora
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: None
+load_best_model_at_end:
+  desc: null
+  value: true
+metric_for_best_model:
+  desc: null
+  value: loss
+greater_is_better:
+  desc: null
+  value: false
+ignore_data_skip:
+  desc: null
+  value: false
+sharded_ddp:
+  desc: null
+  value: '[]'
+fsdp:
+  desc: null
+  value: '[]'
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value: '{''fsdp_min_num_params'': 0, ''xla'': False, ''xla_fsdp_grad_ckpt'': False}'
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: None
+deepspeed:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: None
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value: '[''tensorboard'', ''wandb'']'
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+ddp_broadcast_buffers:
+  desc: null
+  value: None
+dataloader_pin_memory:
+  desc: null
+  value: true
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: None
+hub_model_id:
+  desc: null
+  value: zzz99/output-67b-26k-lora
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: None
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: None
+torch_compile_mode:
+  desc: null
+  value: None
+xpu_backend:
+  desc: null
+  value: None
+cache_dir:
+  desc: null
+  value: None
+model_max_length:
+  desc: null
+  value: 960
+gradient_checkpointing_kwargs:
+  desc: null
+  value: '{''use_reentrant'': False}'
+train_batch_size:
+  desc: null
+  value: 4
+eval_batch_size:
+  desc: null
+  value: 1

wandb/run-20240208_180020-3wdew33h/files/output.log

@@ -0,0 +1,21 @@
+  0%|                                                                                                                              | 0/2 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+Traceback (most recent call last):
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py", line 236, in <module>
+    train()
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py", line 223, in train
+    trainer.train()
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+    return inner_training_loop(
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2665, in training_step
+    self.accelerator.backward(loss)
+  File "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py", line 1853, in backward
+    loss.backward(**kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20240208_180020-3wdew33h/files/requirements.txt

@@ -0,0 +1,419 @@
+absl-py==2.0.0
+accelerate==0.21.0
+aiobotocore==2.7.0
+aiohttp==3.9.1
+aioitertools==0.11.0
+aiosignal==1.3.1
+aiosqlite==0.19.0
+altair==5.2.0
+amazon-codewhisperer-jupyterlab-ext==2.0.1
+amazon-sagemaker-jupyter-scheduler==3.0.5
+annotated-types==0.6.0
+ansi2html==0.0.0
+ansiwrap==0.8.4
+antlr4-python3-runtime==4.9.3
+anyio==3.7.1
+appdirs==1.4.4
+archspec==0.2.2
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+astroid==3.0.1
+asttokens==2.4.1
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+autogluon.common==0.8.2
+autogluon.core==0.8.2
+autogluon.features==0.8.2
+autogluon.multimodal==0.8.2
+autogluon.tabular==0.8.2
+autogluon.timeseries==0.8.2
+autogluon==0.8.2
+autopep8==2.0.4
+autovizwidget==0.21.0
+aws-embedded-metrics==3.2.0
+aws-glue-sessions==1.0.4
+babel==2.13.1
+beautifulsoup4==4.12.2
+binaryornot==0.4.4
+black==23.11.0
+bleach==6.1.0
+blinker==1.7.0
+blis==0.7.10
+boltons==23.0.0
+boto3==1.28.64
+botocore==1.31.64
+brotli==1.0.9
+cached-property==1.5.2
+cachetools==5.3.2
+catalogue==2.0.10
+catboost==1.2.2
+certifi==2023.11.17
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==2.2.1
+colorama==0.4.6
+comm==0.1.4
+conda-libmamba-solver==23.11.1
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.11.0
+confection==0.1.4
+contextlib2==21.6.0
+contourpy==1.2.0
+cookiecutter==2.5.0
+croniter==1.4.1
+cryptography==41.0.7
+cycler==0.12.1
+cymem==2.0.8
+cytoolz==0.12.2
+dash==2.14.2
+dask==2023.12.0
+dataclasses-json==0.6.3
+dataclasses==0.8
+datasets==2.14.7
+debugpy==1.8.0
+decorator==5.1.1
+deepmerge==1.1.0
+defusedxml==0.7.1
+dill==0.3.7
+distributed==2023.12.0
+distro==1.8.0
+docker-pycreds==0.4.0
+docstring-to-markdown==0.13
+entrypoints==0.4
+et-xmlfile==1.1.0
+evaluate==0.4.1
+exceptiongroup==1.2.0
+executing==2.0.1
+faiss==1.7.4
+fastai==2.7.13
+fastcore==1.5.29
+fastdownload==0.0.7
+fastjsonschema==2.19.0
+fastprogress==1.0.3
+filelock==3.13.1
+flake8==6.1.0
+flask==3.0.0
+flatbuffers==23.5.26
+fonttools==4.46.0
+fqdn==1.5.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+future==0.18.3
+gast==0.4.0
+gdown==4.7.1
+gitdb==4.0.11
+gitpython==3.1.40
+gluonts==0.13.7
+gmpy2==2.1.2
+google-api-core==2.15.0
+google-auth-oauthlib==1.0.0
+google-auth==2.25.2
+google-pasta==0.2.0
+googleapis-common-protos==1.62.0
+graphviz==0.20.1
+greenlet==3.0.1
+grpcio==1.54.3
+gssapi==1.8.3
+h5py==3.10.0
+hdijupyterutils==0.21.0
+huggingface-hub==0.19.4
+idna==3.6
+imagecodecs==2023.8.12
+imageio==2.31.5
+importlib-metadata==6.10.0
+importlib-resources==6.1.1
+iniconfig==2.0.0
+ipykernel==6.26.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+isort==5.12.0
+itsdangerous==2.1.2
+jax==0.4.20
+jaxlib==0.4.14
+jedi==0.19.1
+jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpath-ng==1.6.0
+jsonpointer==2.4
+jsonschema==4.17.3
+jupyter-ai-magics==2.6.0
+jupyter-ai==2.6.0
+jupyter-client==8.6.0
+jupyter-console==6.6.3
+jupyter-core==5.5.0
+jupyter-dash==0.4.2
+jupyter-events==0.6.3
+jupyter-lsp==2.2.1
+jupyter-scheduler==2.4.0
+jupyter-server-mathjax==0.2.6
+jupyter-server-proxy==4.1.0
+jupyter-server-terminals==0.5.0
+jupyter-server==2.10.0
+jupyter==1.0.0
+jupyterlab-git==0.50.0
+jupyterlab-lsp==5.0.1
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.24.0
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+keras-preprocessing==1.1.2
+keras==2.12.0
+kiwisolver==1.4.5
+krb5==0.5.1
+langchain==0.0.318
+langcodes==3.3.0
+langsmith==0.0.69
+libmambapy==1.5.3
+lightgbm==3.3.5
+lightning-utilities==0.10.0
+llvmlite==0.41.1
+locket==1.0.0
+markdown-it-py==3.0.0
+markdown==3.5.1
+markupsafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mccabe==0.7.0
+mdurl==0.1.0
+menuinst==2.0.1
+mistune==3.0.2
+ml-dtypes==0.3.1
+mlforecast==0.7.3
+mock==5.1.0
+model-index==0.1.11
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+munkres==1.1.4
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.12.0
+nbdime==4.0.1
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nlpaug==1.1.11
+nltk==3.8.1
+nose==1.3.7
+notebook-shim==0.2.3
+notebook==7.0.6
+nptyping==2.4.1
+numba==0.58.1
+numexpr==2.8.7
+numpy==1.26.2
+oauthlib==3.2.2
+omegaconf==2.2.3
+openai==0.28.1
+openapi-schema-pydantic==1.2.4
+openmim==0.3.7
+openpyxl==3.1.2
+opt-einsum==3.3.0
+ordered-set==4.1.0
+overrides==7.4.0
+packaging==23.2
+pandas-stubs==2.1.1.230928
+pandas==2.1.4
+pandocfilters==1.5.0
+papermill==2.4.0
+parso==0.8.3
+partd==1.4.1
+pathos==0.3.1
+pathspec==0.12.1
+pathtools==0.1.2
+pathy==0.10.2
+patsy==0.5.4
+peft==0.8.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.5.0
+pip==23.3.1
+pkgutil-resolve-name==1.3.10
+platformdirs==4.1.0
+plotly==5.18.0
+pluggy==1.3.0
+ply==3.11
+pox==0.3.3
+ppft==1.7.6.7
+preshed==3.0.9
+prometheus-client==0.19.0
+prompt-toolkit==3.0.41
+protobuf==4.21.12
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pure-sasl==0.6.2
+pyarrow-hotfix==0.6
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pycodestyle==2.11.1
+pycosat==0.6.6
+pycparser==2.21
+pydantic-core==2.14.3
+pydantic==1.10.13
+pydocstyle==6.3.0
+pyflakes==3.1.0
+pygments==2.17.2
+pyhive==0.7.0
+pyjwt==2.8.0
+pylint==3.0.3
+pyopenssl==23.3.0
+pyparsing==3.1.1
+pyqt5-sip==12.12.2
+pyqt5==5.15.9
+pyrsistent==0.20.0
+pysocks==1.7.1
+pyspnego==0.9.1
+pytesseract==0.3.10
+pytest-subtests==0.11.0
+pytest==7.4.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.9.0
+python-slugify==8.0.1
+pytoolconfig==1.2.5
+pytorch-lightning==2.0.9
+pytorch-metric-learning==1.7.3
+pytz==2023.3
+pyu2f==0.1.5
+pywavelets==1.4.1
+pyyaml==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+qtpy==2.4.1
+regex==2023.10.3
+requests-kerberos==0.14.0
+requests-oauthlib==1.3.1
+requests==2.31.0
+responses==0.18.0
+retrying==1.3.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rope==1.11.0
+rsa==4.9
+ruamel.yaml.clib==0.2.7
+ruamel.yaml==0.18.5
+s3transfer==0.7.0
+sacremoses==0.0.53
+safetensors==0.3.3
+sagemaker-headless-execution-driver==0.0.12
+sagemaker-jupyterlab-emr-extension==0.1.9
+sagemaker-jupyterlab-extension-common==0.1.9
+sagemaker-jupyterlab-extension==0.2.0
+sagemaker-kernel-wrapper==0.0.2
+sagemaker-studio-analytics-extension==0.0.21
+sagemaker-studio-sparkmagic-lib==0.1.4
+sagemaker==2.198.0
+sasl==0.3.1
+schema==0.7.5
+scikit-image==0.19.3
+scikit-learn==1.3.2
+scipy==1.11.4
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==1.38.0
+seqeval==1.2.2
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+simpervisor==1.0.0
+sip==6.7.12
+six==1.16.0
+smart-open==5.2.1
+smdebug-rulesconfig==1.0.1
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sparkmagic==0.21.0
+sqlalchemy==1.4.49
+srsly==2.4.8
+stack-data==0.6.2
+statsforecast==1.4.0
+statsmodels==0.14.0
+supervisor==4.2.5
+sympy==1.12
+tabulate==0.9.0
+tblib==1.7.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.0
+tensorboard==2.12.3
+tensorflow-estimator==2.12.0
+tensorflow==2.12.1
+termcolor==2.3.0
+terminado==0.18.0
+text-unidecode==1.3
+textwrap3==0.9.2
+thinc==8.2.1
+threadpoolctl==3.2.0
+thrift-sasl==0.4.3
+thrift==0.19.0
+tifffile==2023.12.9
+tiktoken==0.5.2
+timm==0.9.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.12.3
+toolz==0.12.0
+torch==2.0.0.post200
+torchmetrics==1.0.3
+torchvision==0.15.2a0+072ec57
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.31.0
+truststore==0.8.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+types-pytz==2023.3.1.1
+typing-extensions==4.5.0
+typing-inspect==0.9.0
+typing-utils==0.1.0
+typish==1.9.3
+tzdata==2023.3
+ujson==5.9.0
+unicodedata2==15.1.0
+uri-template==1.3.0
+urllib3==1.26.18
+wandb==0.16.1
+wasabi==1.1.2
+wcwidth==0.2.12
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.1
+whatthepatch==1.0.5
+wheel==0.42.0
+widgetsnbextension==4.0.9
+window-ops==0.0.14
+wrapt==1.16.0
+xgboost==1.7.6
+xxhash==3.4.1
+yapf==0.40.1
+yarl==1.9.3
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.22.0

wandb/run-20240208_180020-3wdew33h/files/wandb-metadata.json

@@ -0,0 +1,1029 @@
+{
+    "os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
+    "python": "3.10.13",
+    "heartbeatAt": "2024-02-08T18:00:21.239172",
+    "startedAt": "2024-02-08T18:00:20.760995",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py",
+    "codePathLocal": null,
+    "host": "default",
+    "username": "sagemaker-user",
+    "executable": "/opt/conda/bin/python3",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 3011.898104166666,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2708.532,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2549.917,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2649.342,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2617.578,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2514.64,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.018,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2514.814,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2823.715,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2444.923,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2450.425,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2405.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.611,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2521.7,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2536.269,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2454.76,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2452.125,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2578.243,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2588.493,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2597.93,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2362.191,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2603.048,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1726.583,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2010.457,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1983.584,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2585.543,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2448.628,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2473.293,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2326.422,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2469.471,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1679.725,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1707.891,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2608.243,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1825.706,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1852.851,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2741.597,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2763.617,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2668.973,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1848.906,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1859.662,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1872.243,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2750.258,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2171.724,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2673.366,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2686.6,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2254.628,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1922.174,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1919.649,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1914.156,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.749,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.574,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.179,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.775,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.087,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.329,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2200.612,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2744.304,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.722,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.792,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.956,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3301.117,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2226.489,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3070.859,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1992.465,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3048.294,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2694.481,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.499,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2794.422,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3292.924,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.232,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.378,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.519,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.987,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.537,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2474.441,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2795.47,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2412.968,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2550.34,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3009.464,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2578.135,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2404.379,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3020.541,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3058.728,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2989.512,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.601,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2714.737,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2694.293,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3261.941,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2586.965,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3294.605,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3260.796,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3295.156,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3253.521,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2832.733,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3264.153,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2767.538,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.784,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2342.793,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2291.218,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2343.318,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2377.449,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2182.822,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.271,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2180.405,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2708.15,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2217.841,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2223.228,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2332.486,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2585.795,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2332.572,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2256.902,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2334.544,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2350.832,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2424.323,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2456.004,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2457.45,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2579.468,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2458.733,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1860.983,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2198.204,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2131.955,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2416.616,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2498.284,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2409.271,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2442.917,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2387.494,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1840.589,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1851.316,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2572.071,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1846.514,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1838.129,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2458.548,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2468.97,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2573.521,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1854.263,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1852.806,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1826.095,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2445.646,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2189.591,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2264.349,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2377.441,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2012.723,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1906.662,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2124.071,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 1895.988,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.029,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2525.577,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.997,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2670.847,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.699,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.175,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2197.86,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2528.763,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3197.634,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3211.437,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3251.423,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2959.085,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2065.145,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2974.061,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2068.626,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2986.901,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2340.617,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.214,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2431.142,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3302.396,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2944.793,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.677,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.69,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2920.67,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3090.867,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2503.478,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2473.681,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2485.275,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2488.179,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2990.001,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2452.206,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2729.116,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3001.225,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.389,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.765,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.125,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2002.363,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2302.227,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3132.468,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2770.449,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3288.126,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.022,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.691,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.894,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2828.863,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.914,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2721.595,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.427,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 32.0,
+            "used": 0.01256561279296875
+        }
+    },
+    "gpu": "NVIDIA A10G",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        }
+    ],
+    "memory": {
+        "total": 747.9597625732422
+    }
+}

wandb/run-20240208_180020-3wdew33h/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 17}}

wandb/run-20240208_180020-3wdew33h/logs/debug-internal.log

@@ -0,0 +1,194 @@
+2024-02-08 18:00:20,766 INFO    StreamThr :1998 [internal.py:wandb_internal():86] W&B internal server running at pid: 1998, started at: 2024-02-08 18:00:20.765271
+2024-02-08 18:00:20,767 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: status
+2024-02-08 18:00:20,770 INFO    WriterThread:1998 [datastore.py:open_for_write():85] open: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/run-3wdew33h.wandb
+2024-02-08 18:00:20,771 DEBUG   SenderThread:1998 [sender.py:send():382] send: header
+2024-02-08 18:00:20,771 DEBUG   SenderThread:1998 [sender.py:send():382] send: run
+2024-02-08 18:00:21,104 INFO    SenderThread:1998 [dir_watcher.py:__init__():211] watching files in: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files
+2024-02-08 18:00:21,104 INFO    SenderThread:1998 [sender.py:_start_run_threads():1136] run started: 3wdew33h with start time 1707415220.764703
+2024-02-08 18:00:21,108 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: check_version
+2024-02-08 18:00:21,108 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: check_version
+2024-02-08 18:00:21,151 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: run_start
+2024-02-08 18:00:21,179 DEBUG   HandlerThread:1998 [system_info.py:__init__():32] System info init
+2024-02-08 18:00:21,179 DEBUG   HandlerThread:1998 [system_info.py:__init__():47] System info init done
+2024-02-08 18:00:21,179 INFO    HandlerThread:1998 [system_monitor.py:start():194] Starting system monitor
+2024-02-08 18:00:21,180 INFO    SystemMonitor:1998 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-02-08 18:00:21,181 INFO    HandlerThread:1998 [system_monitor.py:probe():214] Collecting system info
+2024-02-08 18:00:21,181 INFO    SystemMonitor:1998 [interfaces.py:start():190] Started cpu monitoring
+2024-02-08 18:00:21,182 INFO    SystemMonitor:1998 [interfaces.py:start():190] Started disk monitoring
+2024-02-08 18:00:21,182 INFO    SystemMonitor:1998 [interfaces.py:start():190] Started gpu monitoring
+2024-02-08 18:00:21,184 INFO    SystemMonitor:1998 [interfaces.py:start():190] Started memory monitoring
+2024-02-08 18:00:21,184 INFO    SystemMonitor:1998 [interfaces.py:start():190] Started network monitoring
+2024-02-08 18:00:21,239 DEBUG   HandlerThread:1998 [system_info.py:probe():196] Probing system
+2024-02-08 18:00:21,241 DEBUG   HandlerThread:1998 [gitlib.py:_init_repo():56] git repository is invalid
+2024-02-08 18:00:21,241 DEBUG   HandlerThread:1998 [system_info.py:probe():244] Probing system done
+2024-02-08 18:00:21,241 DEBUG   HandlerThread:1998 [system_monitor.py:probe():223] {'os': 'Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35', 'python': '3.10.13', 'heartbeatAt': '2024-02-08T18:00:21.239172', 'startedAt': '2024-02-08T18:00:20.760995', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py', 'codePathLocal': None, 'host': 'default', 'username': 'sagemaker-user', 'executable': '/opt/conda/bin/python3', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 3011.898104166666, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2708.532, 'min': 0.0, 'max': 0.0}, {'current': 2549.917, 'min': 0.0, 'max': 0.0}, {'current': 2649.342, 'min': 0.0, 'max': 0.0}, {'current': 2617.578, 'min': 0.0, 'max': 0.0}, {'current': 2514.64, 'min': 0.0, 'max': 0.0}, {'current': 3300.018, 'min': 0.0, 'max': 0.0}, {'current': 2514.814, 'min': 0.0, 'max': 0.0}, {'current': 2823.715, 'min': 0.0, 'max': 0.0}, {'current': 2444.923, 'min': 0.0, 'max': 0.0}, {'current': 2450.425, 'min': 0.0, 'max': 0.0}, {'current': 2405.045, 'min': 0.0, 'max': 0.0}, {'current': 3300.611, 'min': 0.0, 'max': 0.0}, {'current': 2521.7, 'min': 0.0, 'max': 0.0}, {'current': 2536.269, 'min': 0.0, 'max': 0.0}, {'current': 2454.76, 'min': 0.0, 'max': 0.0}, {'current': 2452.125, 'min': 0.0, 'max': 0.0}, {'current': 2578.243, 'min': 0.0, 'max': 0.0}, {'current': 2588.493, 'min': 0.0, 'max': 0.0}, {'current': 2597.93, 'min': 0.0, 'max': 0.0}, {'current': 2362.191, 'min': 0.0, 'max': 0.0}, {'current': 2603.048, 'min': 0.0, 'max': 0.0}, {'current': 1726.583, 'min': 0.0, 'max': 0.0}, {'current': 2010.457, 'min': 0.0, 'max': 0.0}, {'current': 1983.584, 'min': 0.0, 'max': 0.0}, {'current': 2585.543, 'min': 0.0, 'max': 0.0}, {'current': 2448.628, 'min': 0.0, 'max': 0.0}, {'current': 2473.293, 'min': 0.0, 'max': 0.0}, {'current': 2326.422, 'min': 0.0, 'max': 0.0}, {'current': 2469.471, 'min': 0.0, 'max': 0.0}, {'current': 1679.725, 'min': 0.0, 'max': 0.0}, {'current': 1707.891, 'min': 0.0, 'max': 0.0}, {'current': 2608.243, 'min': 0.0, 'max': 0.0}, {'current': 1825.706, 'min': 0.0, 'max': 0.0}, {'current': 1852.851, 'min': 0.0, 'max': 0.0}, {'current': 2741.597, 'min': 0.0, 'max': 0.0}, {'current': 2763.617, 'min': 0.0, 'max': 0.0}, {'current': 2668.973, 'min': 0.0, 'max': 0.0}, {'current': 1848.906, 'min': 0.0, 'max': 0.0}, {'current': 1859.662, 'min': 0.0, 'max': 0.0}, {'current': 1872.243, 'min': 0.0, 'max': 0.0}, {'current': 2750.258, 'min': 0.0, 'max': 0.0}, {'current': 2171.724, 'min': 0.0, 'max': 0.0}, {'current': 2673.366, 'min': 0.0, 'max': 0.0}, {'current': 2686.6, 'min': 0.0, 'max': 0.0}, {'current': 2254.628, 'min': 0.0, 'max': 0.0}, {'current': 1922.174, 'min': 0.0, 'max': 0.0}, {'current': 1919.649, 'min': 0.0, 'max': 0.0}, {'current': 1914.156, 'min': 0.0, 'max': 0.0}, {'current': 3299.749, 'min': 0.0, 'max': 0.0}, {'current': 3299.574, 'min': 0.0, 'max': 0.0}, {'current': 3299.179, 'min': 0.0, 'max': 0.0}, {'current': 3300.775, 'min': 0.0, 'max': 0.0}, {'current': 3299.087, 'min': 0.0, 'max': 0.0}, {'current': 3299.329, 'min': 0.0, 'max': 0.0}, {'current': 2200.612, 'min': 0.0, 'max': 0.0}, {'current': 2744.304, 'min': 0.0, 'max': 0.0}, {'current': 3300.722, 'min': 0.0, 'max': 0.0}, {'current': 3299.792, 'min': 0.0, 'max': 0.0}, {'current': 3299.956, 'min': 0.0, 'max': 0.0}, {'current': 3301.117, 'min': 0.0, 'max': 0.0}, {'current': 2226.489, 'min': 0.0, 'max': 0.0}, {'current': 3070.859, 'min': 0.0, 'max': 0.0}, {'current': 1992.465, 'min': 0.0, 'max': 0.0}, {'current': 3048.294, 'min': 0.0, 'max': 0.0}, {'current': 2694.481, 'min': 0.0, 'max': 0.0}, {'current': 3299.499, 'min': 0.0, 'max': 0.0}, {'current': 2794.422, 'min': 0.0, 'max': 0.0}, {'current': 3292.924, 'min': 0.0, 'max': 0.0}, {'current': 3299.232, 'min': 0.0, 'max': 0.0}, {'current': 3297.378, 'min': 0.0, 'max': 0.0}, {'current': 3300.519, 'min': 0.0, 'max': 0.0}, {'current': 3300.987, 'min': 0.0, 'max': 0.0}, {'current': 3298.537, 'min': 0.0, 'max': 0.0}, {'current': 2474.441, 'min': 0.0, 'max': 0.0}, {'current': 2795.47, 'min': 0.0, 'max': 0.0}, {'current': 2412.968, 'min': 0.0, 'max': 0.0}, {'current': 2550.34, 'min': 0.0, 'max': 0.0}, {'current': 3009.464, 'min': 0.0, 'max': 0.0}, {'current': 2578.135, 'min': 0.0, 'max': 0.0}, {'current': 2404.379, 'min': 0.0, 'max': 0.0}, {'current': 3020.541, 'min': 0.0, 'max': 0.0}, {'current': 3058.728, 'min': 0.0, 'max': 0.0}, {'current': 2989.512, 'min': 0.0, 'max': 0.0}, {'current': 3059.601, 'min': 0.0, 'max': 0.0}, {'current': 2714.737, 'min': 0.0, 'max': 0.0}, {'current': 2694.293, 'min': 0.0, 'max': 0.0}, {'current': 3261.941, 'min': 0.0, 'max': 0.0}, {'current': 2586.965, 'min': 0.0, 'max': 0.0}, {'current': 3294.605, 'min': 0.0, 'max': 0.0}, {'current': 3260.796, 'min': 0.0, 'max': 0.0}, {'current': 3295.156, 'min': 0.0, 'max': 0.0}, {'current': 3253.521, 'min': 0.0, 'max': 0.0}, {'current': 2832.733, 'min': 0.0, 'max': 0.0}, {'current': 3264.153, 'min': 0.0, 'max': 0.0}, {'current': 2767.538, 'min': 0.0, 'max': 0.0}, {'current': 3300.784, 'min': 0.0, 'max': 0.0}, {'current': 2342.793, 'min': 0.0, 'max': 0.0}, {'current': 2291.218, 'min': 0.0, 'max': 0.0}, {'current': 2343.318, 'min': 0.0, 'max': 0.0}, {'current': 2377.449, 'min': 0.0, 'max': 0.0}, {'current': 2182.822, 'min': 0.0, 'max': 0.0}, {'current': 3300.271, 'min': 0.0, 'max': 0.0}, {'current': 2180.405, 'min': 0.0, 'max': 0.0}, {'current': 2708.15, 'min': 0.0, 'max': 0.0}, {'current': 2217.841, 'min': 0.0, 'max': 0.0}, {'current': 2223.228, 'min': 0.0, 'max': 0.0}, {'current': 2332.486, 'min': 0.0, 'max': 0.0}, {'current': 2585.795, 'min': 0.0, 'max': 0.0}, {'current': 2332.572, 'min': 0.0, 'max': 0.0}, {'current': 2256.902, 'min': 0.0, 'max': 0.0}, {'current': 2334.544, 'min': 0.0, 'max': 0.0}, {'current': 2350.832, 'min': 0.0, 'max': 0.0}, {'current': 2424.323, 'min': 0.0, 'max': 0.0}, {'current': 2456.004, 'min': 0.0, 'max': 0.0}, {'current': 2457.45, 'min': 0.0, 'max': 0.0}, {'current': 2579.468, 'min': 0.0, 'max': 0.0}, {'current': 2458.733, 'min': 0.0, 'max': 0.0}, {'current': 1860.983, 'min': 0.0, 'max': 0.0}, {'current': 2198.204, 'min': 0.0, 'max': 0.0}, {'current': 2131.955, 'min': 0.0, 'max': 0.0}, {'current': 2416.616, 'min': 0.0, 'max': 0.0}, {'current': 2498.284, 'min': 0.0, 'max': 0.0}, {'current': 2409.271, 'min': 0.0, 'max': 0.0}, {'current': 2442.917, 'min': 0.0, 'max': 0.0}, {'current': 2387.494, 'min': 0.0, 'max': 0.0}, {'current': 1840.589, 'min': 0.0, 'max': 0.0}, {'current': 1851.316, 'min': 0.0, 'max': 0.0}, {'current': 2572.071, 'min': 0.0, 'max': 0.0}, {'current': 1846.514, 'min': 0.0, 'max': 0.0}, {'current': 1838.129, 'min': 0.0, 'max': 0.0}, {'current': 2458.548, 'min': 0.0, 'max': 0.0}, {'current': 2468.97, 'min': 0.0, 'max': 0.0}, {'current': 2573.521, 'min': 0.0, 'max': 0.0}, {'current': 1854.263, 'min': 0.0, 'max': 0.0}, {'current': 1852.806, 'min': 0.0, 'max': 0.0}, {'current': 1826.095, 'min': 0.0, 'max': 0.0}, {'current': 2445.646, 'min': 0.0, 'max': 0.0}, {'current': 2189.591, 'min': 0.0, 'max': 0.0}, {'current': 2264.349, 'min': 0.0, 'max': 0.0}, {'current': 2377.441, 'min': 0.0, 'max': 0.0}, {'current': 2012.723, 'min': 0.0, 'max': 0.0}, {'current': 1906.662, 'min': 0.0, 'max': 0.0}, {'current': 2124.071, 'min': 0.0, 'max': 0.0}, {'current': 1895.988, 'min': 0.0, 'max': 0.0}, {'current': 3300.029, 'min': 0.0, 'max': 0.0}, {'current': 2525.577, 'min': 0.0, 'max': 0.0}, {'current': 3299.997, 'min': 0.0, 'max': 0.0}, {'current': 2670.847, 'min': 0.0, 'max': 0.0}, {'current': 3299.699, 'min': 0.0, 'max': 0.0}, {'current': 3299.175, 'min': 0.0, 'max': 0.0}, {'current': 2197.86, 'min': 0.0, 'max': 0.0}, {'current': 2528.763, 'min': 0.0, 'max': 0.0}, {'current': 3197.634, 'min': 0.0, 'max': 0.0}, {'current': 3211.437, 'min': 0.0, 'max': 0.0}, {'current': 3251.423, 'min': 0.0, 'max': 0.0}, {'current': 2959.085, 'min': 0.0, 'max': 0.0}, {'current': 2065.145, 'min': 0.0, 'max': 0.0}, {'current': 2974.061, 'min': 0.0, 'max': 0.0}, {'current': 2068.626, 'min': 0.0, 'max': 0.0}, {'current': 2986.901, 'min': 0.0, 'max': 0.0}, {'current': 2340.617, 'min': 0.0, 'max': 0.0}, {'current': 3032.214, 'min': 0.0, 'max': 0.0}, {'current': 2431.142, 'min': 0.0, 'max': 0.0}, {'current': 3302.396, 'min': 0.0, 'max': 0.0}, {'current': 2944.793, 'min': 0.0, 'max': 0.0}, {'current': 3299.677, 'min': 0.0, 'max': 0.0}, {'current': 3298.69, 'min': 0.0, 'max': 0.0}, {'current': 2920.67, 'min': 0.0, 'max': 0.0}, {'current': 3090.867, 'min': 0.0, 'max': 0.0}, {'current': 2503.478, 'min': 0.0, 'max': 0.0}, {'current': 2473.681, 'min': 0.0, 'max': 0.0}, {'current': 2485.275, 'min': 0.0, 'max': 0.0}, {'current': 2488.179, 'min': 0.0, 'max': 0.0}, {'current': 2990.001, 'min': 0.0, 'max': 0.0}, {'current': 2452.206, 'min': 0.0, 'max': 0.0}, {'current': 2729.116, 'min': 0.0, 'max': 0.0}, {'current': 3001.225, 'min': 0.0, 'max': 0.0}, {'current': 3299.389, 'min': 0.0, 'max': 0.0}, {'current': 3299.765, 'min': 0.0, 'max': 0.0}, {'current': 3299.125, 'min': 0.0, 'max': 0.0}, {'current': 2002.363, 'min': 0.0, 'max': 0.0}, {'current': 2302.227, 'min': 0.0, 'max': 0.0}, {'current': 3132.468, 'min': 0.0, 'max': 0.0}, {'current': 2770.449, 'min': 0.0, 'max': 0.0}, {'current': 3288.126, 'min': 0.0, 'max': 0.0}, {'current': 3298.022, 'min': 0.0, 'max': 0.0}, {'current': 3298.691, 'min': 0.0, 'max': 0.0}, {'current': 3297.894, 'min': 0.0, 'max': 0.0}, {'current': 2828.863, 'min': 0.0, 'max': 0.0}, {'current': 3299.914, 'min': 0.0, 'max': 0.0}, {'current': 2721.595, 'min': 0.0, 'max': 0.0}, {'current': 3299.427, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 32.0, 'used': 0.01256561279296875}}, 'gpu': 'NVIDIA A10G', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}], 'memory': {'total': 747.9597625732422}}
+2024-02-08 18:00:21,241 INFO    HandlerThread:1998 [system_monitor.py:probe():224] Finished collecting system info
+2024-02-08 18:00:21,241 INFO    HandlerThread:1998 [system_monitor.py:probe():227] Publishing system info
+2024-02-08 18:00:21,241 DEBUG   HandlerThread:1998 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2024-02-08 18:00:21,242 DEBUG   HandlerThread:1998 [system_info.py:_save_pip():68] Saving pip packages done
+2024-02-08 18:00:21,242 DEBUG   HandlerThread:1998 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2024-02-08 18:00:22,105 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/requirements.txt
+2024-02-08 18:00:22,105 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/conda-environment.yaml
+2024-02-08 18:00:35,540 DEBUG   HandlerThread:1998 [system_info.py:_save_conda():87] Saving conda packages done
+2024-02-08 18:00:35,542 INFO    HandlerThread:1998 [system_monitor.py:probe():229] Finished publishing system info
+2024-02-08 18:00:35,545 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:00:35,545 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:00:35,545 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:00:35,545 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:00:35,546 DEBUG   SenderThread:1998 [sender.py:send():382] send: files
+2024-02-08 18:00:35,546 INFO    SenderThread:1998 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now
+2024-02-08 18:00:35,550 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-08 18:00:35,550 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: stop_status
+2024-02-08 18:00:35,557 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:00:35,673 DEBUG   SenderThread:1998 [sender.py:send():382] send: telemetry
+2024-02-08 18:00:35,673 DEBUG   SenderThread:1998 [sender.py:send():382] send: config
+2024-02-08 18:00:35,674 DEBUG   SenderThread:1998 [sender.py:send():382] send: metric
+2024-02-08 18:00:35,674 DEBUG   SenderThread:1998 [sender.py:send():382] send: telemetry
+2024-02-08 18:00:35,674 DEBUG   SenderThread:1998 [sender.py:send():382] send: metric
+2024-02-08 18:00:35,674 WARNING SenderThread:1998 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen)
+2024-02-08 18:00:35,893 INFO    wandb-upload_0:1998 [upload_job.py:push():131] Uploaded file /tmp/tmp_9e35467wandb/s8xuyfes-wandb-metadata.json
+2024-02-08 18:00:36,107 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/conda-environment.yaml
+2024-02-08 18:00:36,107 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/output.log
+2024-02-08 18:00:36,108 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/wandb-metadata.json
+2024-02-08 18:00:36,534 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:00:38,108 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/output.log
+2024-02-08 18:00:38,575 DEBUG   SenderThread:1998 [sender.py:send():382] send: exit
+2024-02-08 18:00:38,575 INFO    SenderThread:1998 [sender.py:send_exit():589] handling exit code: 1
+2024-02-08 18:00:38,575 INFO    SenderThread:1998 [sender.py:send_exit():591] handling runtime: 17
+2024-02-08 18:00:38,575 INFO    SenderThread:1998 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:00:38,575 INFO    SenderThread:1998 [sender.py:send_exit():597] send defer
+2024-02-08 18:00:38,575 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,576 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 0
+2024-02-08 18:00:38,576 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,576 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-02-08 18:00:38,576 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 1
+2024-02-08 18:00:38,576 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,576 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 1
+2024-02-08 18:00:38,576 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,576 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-02-08 18:00:38,576 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 2
+2024-02-08 18:00:38,576 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,576 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 2
+2024-02-08 18:00:38,576 INFO    HandlerThread:1998 [system_monitor.py:finish():203] Stopping system monitor
+2024-02-08 18:00:38,578 INFO    HandlerThread:1998 [interfaces.py:finish():202] Joined cpu monitor
+2024-02-08 18:00:38,578 INFO    HandlerThread:1998 [interfaces.py:finish():202] Joined disk monitor
+2024-02-08 18:00:38,578 DEBUG   SystemMonitor:1998 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-02-08 18:00:38,578 DEBUG   SystemMonitor:1998 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-02-08 18:00:38,578 DEBUG   SystemMonitor:1998 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-02-08 18:00:38,618 INFO    HandlerThread:1998 [interfaces.py:finish():202] Joined gpu monitor
+2024-02-08 18:00:38,618 INFO    HandlerThread:1998 [interfaces.py:finish():202] Joined memory monitor
+2024-02-08 18:00:38,618 INFO    HandlerThread:1998 [interfaces.py:finish():202] Joined network monitor
+2024-02-08 18:00:38,618 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,619 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-02-08 18:00:38,619 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 3
+2024-02-08 18:00:38,619 DEBUG   SenderThread:1998 [sender.py:send():382] send: stats
+2024-02-08 18:00:38,620 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,620 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 3
+2024-02-08 18:00:38,620 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,620 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-02-08 18:00:38,620 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 4
+2024-02-08 18:00:38,620 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,620 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 4
+2024-02-08 18:00:38,620 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,620 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-02-08 18:00:38,620 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 5
+2024-02-08 18:00:38,620 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,621 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 5
+2024-02-08 18:00:38,621 DEBUG   SenderThread:1998 [sender.py:send():382] send: summary
+2024-02-08 18:00:38,633 INFO    SenderThread:1998 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:00:38,633 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,633 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-02-08 18:00:38,633 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 6
+2024-02-08 18:00:38,634 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,634 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 6
+2024-02-08 18:00:38,634 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,634 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-02-08 18:00:38,638 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:00:38,773 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 7
+2024-02-08 18:00:38,773 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:38,773 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 7
+2024-02-08 18:00:38,774 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:38,774 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-02-08 18:00:39,108 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/config.yaml
+2024-02-08 18:00:39,108 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/wandb-summary.json
+2024-02-08 18:00:39,575 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:00:39,685 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 8
+2024-02-08 18:00:39,685 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:00:39,686 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:39,686 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 8
+2024-02-08 18:00:39,686 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:39,686 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-02-08 18:00:39,686 INFO    SenderThread:1998 [job_builder.py:build():298] Attempting to build job artifact
+2024-02-08 18:00:39,688 INFO    SenderThread:1998 [job_builder.py:_get_source_type():439] no source found
+2024-02-08 18:00:39,688 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 9
+2024-02-08 18:00:39,688 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:39,688 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 9
+2024-02-08 18:00:39,688 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:39,688 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-02-08 18:00:39,688 INFO    SenderThread:1998 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-02-08 18:00:40,109 INFO    Thread-12 :1998 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/output.log
+2024-02-08 18:00:40,109 INFO    SenderThread:1998 [dir_watcher.py:finish():388] scan: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files
+2024-02-08 18:00:40,109 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/config.yaml config.yaml
+2024-02-08 18:00:40,109 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/requirements.txt requirements.txt
+2024-02-08 18:00:40,109 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/conda-environment.yaml conda-environment.yaml
+2024-02-08 18:00:40,109 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/wandb-metadata.json wandb-metadata.json
+2024-02-08 18:00:40,110 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/output.log output.log
+2024-02-08 18:00:40,111 INFO    SenderThread:1998 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/wandb-summary.json wandb-summary.json
+2024-02-08 18:00:40,111 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 10
+2024-02-08 18:00:40,112 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:40,112 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 10
+2024-02-08 18:00:40,114 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:40,114 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-02-08 18:00:40,114 INFO    SenderThread:1998 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:00:40,385 INFO    wandb-upload_0:1998 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/config.yaml
+2024-02-08 18:00:40,437 INFO    wandb-upload_3:1998 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/output.log
+2024-02-08 18:00:40,470 INFO    wandb-upload_2:1998 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/conda-environment.yaml
+2024-02-08 18:00:40,560 INFO    wandb-upload_1:1998 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/requirements.txt
+2024-02-08 18:00:40,575 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:00:40,576 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:00:40,674 INFO    wandb-upload_4:1998 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/files/wandb-summary.json
+2024-02-08 18:00:40,875 INFO    Thread-11 (_thread_body):1998 [sender.py:transition_state():617] send defer: 11
+2024-02-08 18:00:40,875 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:40,875 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 11
+2024-02-08 18:00:40,875 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:40,875 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-02-08 18:00:40,875 INFO    SenderThread:1998 [file_pusher.py:join():181] waiting for file pusher
+2024-02-08 18:00:40,876 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 12
+2024-02-08 18:00:40,876 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:40,876 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 12
+2024-02-08 18:00:40,876 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:40,876 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-02-08 18:00:40,876 INFO    SenderThread:1998 [file_stream.py:finish():595] file stream finish called
+2024-02-08 18:00:40,954 INFO    SenderThread:1998 [file_stream.py:finish():599] file stream finish is done
+2024-02-08 18:00:40,954 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 13
+2024-02-08 18:00:40,954 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:40,954 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 13
+2024-02-08 18:00:40,954 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:40,954 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-02-08 18:00:40,954 INFO    SenderThread:1998 [sender.py:transition_state():617] send defer: 14
+2024-02-08 18:00:40,955 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:00:40,955 INFO    HandlerThread:1998 [handler.py:handle_request_defer():172] handle defer: 14
+2024-02-08 18:00:40,955 DEBUG   SenderThread:1998 [sender.py:send():382] send: final
+2024-02-08 18:00:40,955 DEBUG   SenderThread:1998 [sender.py:send():382] send: footer
+2024-02-08 18:00:40,955 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:00:40,955 INFO    SenderThread:1998 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-02-08 18:00:40,955 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:00:40,956 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:00:40,956 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:00:40,956 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: server_info
+2024-02-08 18:00:40,956 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:00:40,957 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: server_info
+2024-02-08 18:00:40,959 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: get_summary
+2024-02-08 18:00:40,959 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: sampled_history
+2024-02-08 18:00:40,959 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:00:40,959 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: job_info
+2024-02-08 18:00:41,013 DEBUG   SenderThread:1998 [sender.py:send_request():409] send_request: job_info
+2024-02-08 18:00:41,013 INFO    MainThread:1998 [wandb_run.py:_footer_history_summary_info():3837] rendering history
+2024-02-08 18:00:41,013 INFO    MainThread:1998 [wandb_run.py:_footer_history_summary_info():3869] rendering summary
+2024-02-08 18:00:41,013 INFO    MainThread:1998 [wandb_run.py:_footer_sync_info():3796] logging synced files
+2024-02-08 18:00:41,014 DEBUG   HandlerThread:1998 [handler.py:handle_request():146] handle_request: shutdown
+2024-02-08 18:00:41,014 INFO    HandlerThread:1998 [handler.py:finish():866] shutting down handler
+2024-02-08 18:00:41,960 INFO    WriterThread:1998 [datastore.py:close():294] close: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/run-3wdew33h.wandb
+2024-02-08 18:00:42,013 INFO    SenderThread:1998 [sender.py:finish():1548] shutting down sender
+2024-02-08 18:00:42,013 INFO    SenderThread:1998 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:00:42,013 INFO    SenderThread:1998 [file_pusher.py:join():181] waiting for file pusher

wandb/run-20240208_180020-3wdew33h/logs/debug.log

@@ -0,0 +1,30 @@
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Configure stats pid to 1738
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/.config/wandb/settings
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/output-7b-26k-lora/wandb/settings
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-08 18:00:20,762 WARNING MainThread:1738 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/home/sagemaker-user/lora_finetuning_push_to_hub_save_local.py', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local.py'}
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:_log_setup():524] Logging user logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/logs/debug.log
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:_log_setup():525] Logging internal logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_180020-3wdew33h/logs/debug-internal.log
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:init():564] calling init triggers
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+config: {}
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:init():614] starting backend
+2024-02-08 18:00:20,762 INFO    MainThread:1738 [wandb_init.py:init():618] setting up manager
+2024-02-08 18:00:20,763 INFO    MainThread:1738 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-08 18:00:20,764 INFO    MainThread:1738 [wandb_init.py:init():624] backend started and connected
+2024-02-08 18:00:20,766 INFO    MainThread:1738 [wandb_init.py:init():716] updated telemetry
+2024-02-08 18:00:20,767 INFO    MainThread:1738 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-08 18:00:21,108 INFO    MainThread:1738 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-08 18:00:21,147 INFO    MainThread:1738 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.3 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-02-08 18:00:21,147 INFO    MainThread:1738 [wandb_init.py:init():800] starting run threads in backend
+2024-02-08 18:00:35,551 INFO    MainThread:1738 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-08 18:00:35,551 INFO    MainThread:1738 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-08 18:00:35,551 INFO    MainThread:1738 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-08 18:00:35,551 INFO    MainThread:1738 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-08 18:00:35,552 INFO    MainThread:1738 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-08 18:00:35,554 INFO    MainThread:1738 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 32256, 'max_position_embeddings': 16384, 'hidden_size': 4096, 'intermediate_size': 11008, 'num_hidden_layers': 32, 'num_attention_heads': 32, 'num_key_value_heads': 32, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_scaling': {'factor': 4.0, 'type': 'linear'}, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 32013, 'pad_token_id': 0, 'eos_token_id': 32021, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'deepseek-ai/deepseek-coder-6.7b-instruct', 'transformers_version': '4.31.0', 'attention_bias': False, 'model_type': 'llama', 'rope_theta': 100000, 'output_dir': 'output-67b-26k-lora', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'output-67b-26k-lora/runs/Feb08_17-59-25_default', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': 100, 'save_safetensors': False, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'ddp_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 4, 'past_index': -1, 'run_name': 'output-67b-26k-lora', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': True, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'ddp_broadcast_buffers': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'zzz99/output-67b-26k-lora', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'xpu_backend': 'None', 'cache_dir': 'None', 'model_max_length': 960, 'gradient_checkpointing_kwargs': "{'use_reentrant': False}", 'train_batch_size': 4, 'eval_batch_size': 1}
+2024-02-08 18:00:42,128 WARNING MsgRouterThr:1738 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240208_184521-kk66dbgv/files/conda-environment.yaml

@@ -0,0 +1,662 @@
+name: base
+channels:
+  - file:///tmp/local_conda_forge_artifacts
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - _py-xgboost-mutex=2.0=gpu_0
+  - absl-py=2.0.0=pyhd8ed1ab_0
+  - accelerate=0.21.0=pyhd8ed1ab_0
+  - aiobotocore=2.7.0=pyhd8ed1ab_1
+  - aiohttp=3.9.1=py310h2372a71_0
+  - aioitertools=0.11.0=pyhd8ed1ab_0
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - aiosqlite=0.19.0=pyhd8ed1ab_0
+  - alsa-lib=1.2.10=hd590300_0
+  - altair=5.2.0=pyhd8ed1ab_0
+  - amazon-codewhisperer-jupyterlab-ext=2.0.1=pyhd8ed1ab_0
+  - amazon-sagemaker-jupyter-scheduler=3.0.5=pyhd8ed1ab_0
+  - ansi2html=1.8.0=py310hff52083_2
+  - ansiwrap=0.8.4=py_0
+  - antlr-python-runtime=4.9.3=pyhd8ed1ab_1
+  - anyio=3.7.1=pyhd8ed1ab_0
+  - aom=3.5.0=h27087fc_0
+  - appdirs=1.4.4=pyh9f0ad1d_0
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - astroid=3.0.1=py310hff52083_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - astunparse=1.6.3=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - async-timeout=4.0.3=pyhd8ed1ab_0
+  - atk-1.0=2.38.0=hd4edc92_1
+  - attr=2.5.1=h166bdaf_1
+  - attrs=23.1.0=pyh71513ae_1
+  - autogluon=0.8.2=pyhd8ed1ab_4
+  - autogluon.common=0.8.2=pyhd8ed1ab_4
+  - autogluon.core=0.8.2=light_py310h2d11d36_6
+  - autogluon.features=0.8.2=pyhd8ed1ab_3
+  - autogluon.multimodal=0.8.2=pyha770c72_5
+  - autogluon.tabular=0.8.2=pyha770c72_4
+  - autogluon.timeseries=0.8.2=pyhd8ed1ab_4
+  - autopep8=2.0.4=pyhd8ed1ab_0
+  - autovizwidget=0.21.0=pyh1a96a4e_0
+  - aws-c-auth=0.7.3=h28f7589_1
+  - aws-c-cal=0.6.1=hc309b26_1
+  - aws-c-common=0.9.0=hd590300_0
+  - aws-c-compression=0.2.17=h4d4d85c_2
+  - aws-c-event-stream=0.3.1=h2e3709c_4
+  - aws-c-http=0.7.11=h00aa349_4
+  - aws-c-io=0.13.32=he9a53bd_1
+  - aws-c-mqtt=0.9.3=hb447be9_1
+  - aws-c-s3=0.3.14=hf3aad02_1
+  - aws-c-sdkutils=0.1.12=h4d4d85c_1
+  - aws-checksums=0.1.17=h4d4d85c_1
+  - aws-crt-cpp=0.21.0=hb942446_5
+  - aws-embedded-metrics=3.2.0=pyhd8ed1ab_0
+  - aws-glue-sessions=1.0.4=pyhd8ed1ab_0
+  - aws-sdk-cpp=1.10.57=h85b1a90_19
+  - babel=2.13.1=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.2=pyha770c72_0
+  - binaryornot=0.4.4=py_1
+  - black=23.11.0=py310hff52083_0
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - blinker=1.7.0=pyhd8ed1ab_0
+  - blosc=1.21.5=h0f2a231_0
+  - boltons=23.0.0=pyhd8ed1ab_0
+  - boto3=1.28.64=pyhd8ed1ab_0
+  - botocore=1.31.64=pyhd8ed1ab_0
+  - brotli=1.0.9=h166bdaf_9
+  - brotli-bin=1.0.9=h166bdaf_9
+  - brotli-python=1.0.9=py310hd8f1fbe_9
+  - brunsli=0.1=h9c3ff4c_0
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.23.0=hd590300_0
+  - c-blosc2=2.11.3=hb4ffafa_0
+  - ca-certificates=2023.11.17=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cachetools=5.3.2=pyhd8ed1ab_0
+  - cairo=1.18.0=h3faef2a_0
+  - catalogue=2.0.10=py310hff52083_0
+  - catboost=1.2.2=py310hff52083_0
+  - certifi=2023.11.17=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - chardet=5.2.0=py310hff52083_1
+  - charls=2.4.2=h59595ed_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - click=8.1.7=unix_pyh707e725_0
+  - cloudpathlib=0.16.0=pyhd8ed1ab_0
+  - cloudpickle=2.2.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - conda=23.11.0=py310hff52083_1
+  - conda-libmamba-solver=23.11.1=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - confection=0.1.4=py310h17c5347_0
+  - contextlib2=21.6.0=pyhd8ed1ab_0
+  - contourpy=1.2.0=py310hd41b1e2_0
+  - cookiecutter=2.5.0=pyhca7485f_0
+  - croniter=1.4.1=pyhd8ed1ab_0
+  - cryptography=41.0.7=py310hb8475ec_1
+  - cuda-version=11.8=h70ddcb2_2
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudnn=8.8.0.121=hcdd5f01_4
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - cymem=2.0.8=py310hc6cd4ac_1
+  - cyrus-sasl=2.1.27=h54b06d7_7
+  - cython-blis=0.7.10=py310h1f7b6fc_2
+  - cytoolz=0.12.2=py310h2372a71_1
+  - dash=2.14.2=pyhd8ed1ab_0
+  - dask-core=2023.12.0=pyhd8ed1ab_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - dataclasses-json=0.6.3=pyhd8ed1ab_0
+  - datasets=2.14.7=pyhd8ed1ab_0
+  - dav1d=1.2.1=hd590300_0
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.8.0=py310hc6cd4ac_1
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - deepmerge=1.1.0=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - dill=0.3.7=pyhd8ed1ab_0
+  - distributed=2023.12.0=pyhd8ed1ab_0
+  - distro=1.8.0=pyhd8ed1ab_0
+  - docker-pycreds=0.4.0=py_0
+  - docstring-to-markdown=0.13=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - et_xmlfile=1.1.0=pyhd8ed1ab_0
+  - evaluate=0.4.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.5.0=hcb278e6_1
+  - faiss=1.7.4=py310h9ed8947_0_cpu
+  - faiss-cpu=1.7.4=h718b53a_0
+  - fastai=2.7.13=pyhd8ed1ab_0
+  - fastcore=1.5.29=pyhd8ed1ab_0
+  - fastdownload=0.0.7=pyhd8ed1ab_0
+  - fastprogress=1.0.3=pyhd8ed1ab_0
+  - filelock=3.13.1=pyhd8ed1ab_0
+  - flake8=6.1.0=pyhd8ed1ab_0
+  - flask=3.0.0=pyhd8ed1ab_0
+  - flatbuffers=23.5.26=h59595ed_1
+  - fmt=10.1.1=h00ab1b0_1
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_1
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.46.0=py310h2372a71_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - frozenlist=1.4.0=py310h2372a71_1
+  - fsspec=2023.6.0=pyh1a96a4e_0
+  - future=0.18.3=pyhd8ed1ab_0
+  - gast=0.4.0=pyh9f0ad1d_0
+  - gdk-pixbuf=2.42.10=h6b639ba_2
+  - gdown=4.7.1=pyhd8ed1ab_0
+  - gettext=0.21.1=h27087fc_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h0b41bf4_3
+  - gitdb=4.0.11=pyhd8ed1ab_0
+  - gitpython=3.1.40=pyhd8ed1ab_0
+  - glib=2.78.3=hfc55251_0
+  - glib-tools=2.78.3=hfc55251_0
+  - glog=0.6.0=h6f12383_0
+  - gluonts=0.13.7=pyhd8ed1ab_0
+  - gmp=6.3.0=h59595ed_0
+  - gmpy2=2.1.2=py310h3ec546c_1
+  - google-api-core=2.15.0=pyhd8ed1ab_0
+  - google-auth=2.25.2=pyhca7485f_0
+  - google-auth-oauthlib=1.0.0=pyhd8ed1ab_1
+  - google-pasta=0.2.0=pyh8c360ce_0
+  - googleapis-common-protos=1.62.0=pyhd8ed1ab_0
+  - graphite2=1.3.13=h58526e2_1001
+  - graphviz=8.1.0=h28d9a01_0
+  - grpcio=1.54.3=py310heca2aa9_0
+  - gst-plugins-base=1.22.7=h8e1006c_1
+  - gstreamer=1.22.7=h98fc4e7_1
+  - gtk2=2.24.33=h90689f9_2
+  - gts=0.7.6=h977cf35_4
+  - h5py=3.10.0=nompi_py310ha2ad45a_100
+  - harfbuzz=8.3.0=h3d44ed6_0
+  - hdf5=1.14.2=nompi_h4f84152_100
+  - hdijupyterutils=0.21.0=pyh1a96a4e_0
+  - huggingface_hub=0.19.4=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - imagecodecs=2023.8.12=py310hc929067_0
+  - imageio=2.31.5=pyh8c1a49c_0
+  - importlib-metadata=6.10.0=pyha770c72_0
+  - importlib_metadata=6.10.0=hd8ed1ab_0
+  - importlib_resources=6.1.1=pyhd8ed1ab_0
+  - iniconfig=2.0.0=pyhd8ed1ab_0
+  - ipykernel=6.26.0=pyhf8b6a83_0
+  - ipython=8.18.1=pyh707e725_3
+  - ipywidgets=8.1.1=pyhd8ed1ab_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - isort=5.12.0=pyhd8ed1ab_1
+  - itsdangerous=2.1.2=pyhd8ed1ab_0
+  - jax=0.4.20=pyhd8ed1ab_0
+  - jaxlib=0.4.14=cpu_py310h67d73b5_1
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.3.2=pyhd8ed1ab_0
+  - json5=0.9.14=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpath-ng=1.6.0=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.17.3=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.17.3=pyhd8ed1ab_0
+  - jupyter=1.0.0=pyhd8ed1ab_10
+  - jupyter-ai=2.6.0=pyhd8ed1ab_2
+  - jupyter-ai-magics=2.6.0=pyhd8ed1ab_1
+  - jupyter-dash=0.4.2=pyhd8ed1ab_1
+  - jupyter-lsp=2.2.1=pyhd8ed1ab_0
+  - jupyter-server-mathjax=0.2.6=pyh5bfe37b_1
+  - jupyter-server-proxy=4.1.0=pyhd8ed1ab_0
+  - jupyter_client=8.6.0=pyhd8ed1ab_0
+  - jupyter_console=6.6.3=pyhd8ed1ab_0
+  - jupyter_core=5.5.0=py310hff52083_0
+  - jupyter_events=0.6.3=pyhd8ed1ab_1
+  - jupyter_scheduler=2.4.0=pyha770c72_1
+  - jupyter_server=2.10.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0
+  - jupyterlab=4.0.9=pyhd8ed1ab_0
+  - jupyterlab-git=0.50.0=pyhd8ed1ab_0
+  - jupyterlab-lsp=5.0.1=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0
+  - jupyterlab_server=2.24.0=pyhd8ed1ab_0
+  - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0
+  - jxrlib=1.1=h7f98852_2
+  - keras=2.12.0=pyhd8ed1ab_0
+  - keras-preprocessing=1.1.2=pyhd8ed1ab_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py310hd41b1e2_1
+  - lame=3.100=h166bdaf_1003
+  - langchain=0.0.318=pyhd8ed1ab_0
+  - langcodes=3.3.0=pyhd8ed1ab_0
+  - langsmith=0.0.69=pyhd8ed1ab_0
+  - lcms2=2.15=haa2dc70_1
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20230125.3=cxx17_h59595ed_0
+  - libaec=1.1.2=h59595ed_1
+  - libarchive=3.7.2=h039dbb9_0
+  - libarrow=12.0.1=hb87d912_8_cpu
+  - libavif=0.11.1=h8182462_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libbrotlicommon=1.0.9=h166bdaf_9
+  - libbrotlidec=1.0.9=h166bdaf_9
+  - libbrotlienc=1.0.9=h166bdaf_9
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=16_linux64_mkl
+  - libclang=15.0.7=default_hb11cfb5_4
+  - libclang13=15.0.7=default_ha2b6cf4_4
+  - libcrc32c=1.1.2=h9c3ff4c_0
+  - libcups=2.3.3=h4637d8d_4
+  - libcurl=8.4.0=hca28451_0
+  - libdeflate=1.18=h0b41bf4_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.5.0=hcb278e6_1
+  - libfaiss=1.7.4=hf47d654_0_cpu
+  - libfaiss-avx2=1.7.4=h1234567_0_cpu
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgcrypt=1.10.3=hd590300_0
+  - libgd=2.3.3=h74d50f4_7
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libglib=2.78.3=h783c2da_0
+  - libgoogle-cloud=2.12.0=hac9eb74_1
+  - libgpg-error=1.47=h71f35ed_0
+  - libgrpc=1.54.3=hb20ce57_0
+  - libhwloc=2.9.3=default_h554bfaf_1009
+  - libiconv=1.17=hd590300_1
+  - libjpeg-turbo=2.1.5.1=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libllvm14=14.0.6=hcd5def8_4
+  - libllvm15=15.0.7=h5cf9203_3
+  - libmagma=2.7.1=hc72dce7_6
+  - libmagma_sparse=2.7.1=h8354cda_6
+  - libmamba=1.5.3=had39da4_2
+  - libmambapy=1.5.3=py310h39ff949_2
+  - libnghttp2=1.58.0=h47da74e_0
+  - libnsl=2.0.1=hd590300_0
+  - libntlm=1.4=h7f98852_1002
+  - libnuma=2.0.16=h0b41bf4_1
+  - libogg=1.3.4=h7f98852_1
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.39=h753d276_0
+  - libpq=15.4=hfc447b1_2
+  - libprotobuf=3.21.12=hfc55251_2
+  - librsvg=2.56.3=h98fae49_0
+  - libsecret=0.18.8=h329b89f_2
+  - libsentencepiece=0.1.99=h180e1df_0
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libsystemd0=255=h3516f8a_0
+  - libthrift=0.18.1=h8fd135c_2
+  - libtiff=4.5.1=h8b53f26_1
+  - libtool=2.4.7=h27087fc_0
+  - libutf8proc=2.8.0=h166bdaf_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libuv=1.46.0=hd590300_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp=1.3.1=hbf2b3c1_0
+  - libwebp-base=1.3.1=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxgboost=1.7.6=cuda118_hd3b444d_6
+  - libxkbcommon=1.6.0=h5d7e998_0
+  - libxml2=2.11.6=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - libzopfli=1.0.3=h9c3ff4c_0
+  - lightgbm=3.3.5=py310heca2aa9_0
+  - lightning-utilities=0.10.0=pyhd8ed1ab_0
+  - llvm-openmp=17.0.6=h4dfa4b3_0
+  - llvmlite=0.41.1=py310h1b8f574_0
+  - locket=1.0.0=pyhd8ed1ab_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - magma=2.7.1=ha770c72_6
+  - markdown=3.5.1=pyhd8ed1ab_0
+  - markdown-it-py=3.0.0=pyhd8ed1ab_0
+  - markupsafe=2.1.3=py310h2372a71_1
+  - marshmallow=3.20.1=pyhd8ed1ab_0
+  - matplotlib=3.8.2=py310hff52083_0
+  - matplotlib-base=3.8.2=py310h62c0568_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mccabe=0.7.0=pyhd8ed1ab_0
+  - mdurl=0.1.0=pyhd8ed1ab_0
+  - menuinst=2.0.1=py310hff52083_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2022.2.1=h84fe81f_16997
+  - ml_dtypes=0.3.1=py310hcc13569_2
+  - mlforecast=0.7.3=pyhd8ed1ab_0
+  - mock=5.1.0=pyhd8ed1ab_0
+  - model-index=0.1.11=pyhd8ed1ab_0
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h9458935_0
+  - mpg123=1.32.3=h59595ed_0
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - msgpack-python=1.0.7=py310hd41b1e2_0
+  - multidict=6.0.4=py310h2372a71_1
+  - multiprocess=0.70.15=py310h2372a71_1
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.10=py310hc6cd4ac_1
+  - mypy_extensions=1.0.0=pyha770c72_0
+  - mysql-common=8.0.33=hf1915f5_6
+  - mysql-libs=8.0.33=hca2cd23_6
+  - nbclient=0.8.0=pyhd8ed1ab_0
+  - nbconvert=7.12.0=pyhd8ed1ab_0
+  - nbconvert-core=7.12.0=pyhd8ed1ab_0
+  - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0
+  - nbdime=4.0.1=pyhd8ed1ab_0
+  - nbformat=5.9.2=pyhd8ed1ab_0
+  - nccl=2.19.4.1=h6103f9b_0
+  - ncurses=6.4=h59595ed_2
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - networkx=3.2.1=pyhd8ed1ab_0
+  - nlpaug=1.1.11=pyhd8ed1ab_1
+  - nltk=3.8.1=pyhd8ed1ab_0
+  - nodejs=18.18.2=hb753e55_1
+  - nose=1.3.7=py_1006
+  - notebook=7.0.6=pyhd8ed1ab_0
+  - notebook-shim=0.2.3=pyhd8ed1ab_0
+  - nptyping=2.4.1=pyhd8ed1ab_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.95=h1d7d5a4_0
+  - numba=0.58.1=py310h7dc5dd1_0
+  - numexpr=2.8.7=mkl_py310h5e07f1c_4
+  - numpy=1.26.2=py310hb13e2d6_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - omegaconf=2.2.3=pyhd8ed1ab_0
+  - openai=0.28.1=pyhd8ed1ab_0
+  - openapi-schema-pydantic=1.2.4=pyhd8ed1ab_0
+  - openjpeg=2.5.0=hfec8fc6_2
+  - openmim=0.3.7=pyhd8ed1ab_0
+  - openpyxl=3.1.2=py310h2372a71_1
+  - openssl=3.1.4=hd590300_0
+  - opt-einsum=3.3.0=hd8ed1ab_2
+  - opt_einsum=3.3.0=pyhc1e730c_2
+  - orc=1.9.0=h2f23424_1
+  - ordered-set=4.1.0=pyhd8ed1ab_0
+  - overrides=7.4.0=pyhd8ed1ab_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pandas=2.1.4=py310hcc13569_0
+  - pandas-stubs=2.1.1.230928=pyhd8ed1ab_1
+  - pandoc=3.1.3=h32600fe_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - pango=1.50.14=ha41ecd1_2
+  - papermill=2.4.0=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - partd=1.4.1=pyhd8ed1ab_0
+  - pathos=0.3.1=pyhd8ed1ab_0
+  - pathspec=0.12.1=pyhd8ed1ab_0
+  - pathtools=0.1.2=py_1
+  - pathy=0.10.2=pyhd8ed1ab_0
+  - patsy=0.5.4=pyhd8ed1ab_0
+  - pcre2=10.42=hcad00b1_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=9.5.0=py310h582fbeb_1
+  - pip=23.3.1=pyhd8ed1ab_0
+  - pixman=0.42.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - plotly=5.18.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - ply=3.11=py_1
+  - pox=0.3.3=pyhd8ed1ab_0
+  - ppft=1.7.6.7=pyhd8ed1ab_0
+  - preshed=3.0.9=py310hc6cd4ac_1
+  - prometheus_client=0.19.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.41=pyha770c72_0
+  - prompt_toolkit=3.0.41=hd8ed1ab_0
+  - protobuf=4.21.12=py310heca2aa9_0
+  - psutil=5.9.5=py310h2372a71_1
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio-client=16.1=hb77b528_5
+  - pure-sasl=0.6.2=pyhd8ed1ab_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - py-xgboost=1.7.6=cuda118_py310h9840055_6
+  - py-xgboost-gpu=1.7.6=py310h9840055_6
+  - pyarrow=12.0.1=py310h0576679_8_cpu
+  - pyarrow-hotfix=0.6=pyhd8ed1ab_0
+  - pyasn1=0.5.1=pyhd8ed1ab_0
+  - pyasn1-modules=0.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycodestyle=2.11.1=pyhd8ed1ab_0
+  - pycosat=0.6.6=py310h2372a71_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydocstyle=6.3.0=pyhd8ed1ab_0
+  - pyflakes=3.1.0=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pyhive=0.7.0=pyhd8ed1ab_0
+  - pyjwt=2.8.0=pyhd8ed1ab_0
+  - pylint=3.0.3=pyhd8ed1ab_0
+  - pyopenssl=23.3.0=pyhd8ed1ab_0
+  - pyparsing=3.1.1=pyhd8ed1ab_0
+  - pyqt=5.15.9=py310h04931ad_5
+  - pyqt5-sip=12.12.2=py310hc6cd4ac_5
+  - pyrsistent=0.20.0=py310h2372a71_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - pyspnego=0.9.1=py310h2372a71_2
+  - pytesseract=0.3.10=pyhd8ed1ab_0
+  - pytest=7.4.3=pyhd8ed1ab_0
+  - pytest-subtests=0.11.0=pyhd8ed1ab_0
+  - python=3.10.13=hd12c33a_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.0=pyhd8ed1ab_0
+  - python-flatbuffers=23.5.26=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=pyh22cad53_0
+  - python-gssapi=1.8.3=py310h98feede_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python-lsp-jsonrpc=1.1.2=pyhd8ed1ab_0
+  - python-lsp-server=1.9.0=pyhd8ed1ab_0
+  - python-lsp-server-base=1.9.0=pyhd8ed1ab_0
+  - python-slugify=8.0.1=pyhd8ed1ab_2
+  - python-tzdata=2023.3=pyhd8ed1ab_0
+  - python-xxhash=3.4.1=py310h2372a71_0
+  - python_abi=3.10=4_cp310
+  - pytoolconfig=1.2.5=pyhd8ed1ab_0
+  - pytorch=2.0.0=cuda112py310he33e0d6_200
+  - pytorch-gpu=2.0.0=cuda112py310h9871d0b_200
+  - pytorch-lightning=2.0.9=pyhd8ed1ab_0
+  - pytorch-metric-learning=1.7.3=pyhd8ed1ab_0
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywavelets=1.4.1=py310h1f7b6fc_1
+  - pyyaml=6.0.1=py310h2372a71_1
+  - pyzmq=25.1.2=py310h795f18f_0
+  - qt-main=5.15.8=hc47bfe8_16
+  - qtconsole-base=5.5.1=pyha770c72_0
+  - qtpy=2.4.1=pyhd8ed1ab_0
+  - rdma-core=28.9=h59595ed_1
+  - re2=2023.03.02=h8c504da_0
+  - readline=8.2=h8228510_1
+  - regex=2023.10.3=py310h2372a71_0
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - requests-kerberos=0.14.0=pyhd8ed1ab_1
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - retrying=1.3.3=py_2
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rich=13.7.0=pyhd8ed1ab_0
+  - ripgrep=14.0.3=he8a937b_1
+  - rope=1.11.0=pyhd8ed1ab_1
+  - rsa=4.9=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py310h2372a71_0
+  - ruamel.yaml.clib=0.2.7=py310h2372a71_2
+  - s2n=1.3.49=h06160fa_0
+  - s3transfer=0.7.0=pyhd8ed1ab_1
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - safetensors=0.3.3=py310hcb5633a_1
+  - sagemaker-code-editor=1.83.1=ha1e3dc1_3
+  - sagemaker-headless-execution-driver=0.0.12=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-emr-extension=0.1.9=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension=0.2.0=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension-common=0.1.9=pyhd8ed1ab_0
+  - sagemaker-kernel-wrapper=0.0.2=pyhd8ed1ab_0
+  - sagemaker-python-sdk=2.198.0=pyhd8ed1ab_0
+  - sagemaker-studio-analytics-extension=0.0.21=pyhd8ed1ab_0
+  - sagemaker-studio-sparkmagic-lib=0.1.4=pyhd8ed1ab_0
+  - sasl=0.3.1=py310hd8f1fbe_2
+  - schema=0.7.5=pyhd8ed1ab_1
+  - scikit-image=0.19.3=py310h769672d_2
+  - scikit-learn=1.3.2=py310h1fdf081_2
+  - scipy=1.11.4=py310hb13e2d6_0
+  - send2trash=1.8.2=pyh41d4057_0
+  - sentencepiece=0.1.99=hff52083_0
+  - sentencepiece-python=0.1.99=py310hebdb9f0_0
+  - sentencepiece-spm=0.1.99=h180e1df_0
+  - sentry-sdk=1.38.0=pyhd8ed1ab_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setproctitle=1.3.3=py310h2372a71_0
+  - setuptools=68.2.2=pyhd8ed1ab_0
+  - shellingham=1.5.4=pyhd8ed1ab_0
+  - simpervisor=1.0.0=pyhd8ed1ab_0
+  - sip=6.7.12=py310hc6cd4ac_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sleef=3.5.1=h9b69904_2
+  - smart_open=5.2.1=pyhd8ed1ab_0
+  - smdebug-rulesconfig=1.0.1=pyhd3deb0d_1
+  - smmap=5.0.0=pyhd8ed1ab_0
+  - snappy=1.1.10=h9fff704_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - snowballstemmer=2.2.0=pyhd8ed1ab_0
+  - sortedcontainers=2.4.0=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - spacy=3.7.2=py310hcb52e73_0
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.5=pyhd8ed1ab_0
+  - sparkmagic=0.21.0=pyhd8ed1ab_0
+  - srsly=2.4.8=py310hc6cd4ac_1
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - statsforecast=1.4.0=pyhd8ed1ab_0
+  - statsmodels=0.14.0=py310h1f7b6fc_3
+  - supervisor=4.2.5=py310hff52083_2
+  - sympy=1.12=pypyh9d50eac_103
+  - tabulate=0.9.0=pyhd8ed1ab_1
+  - tbb=2021.11.0=h00ab1b0_0
+  - tblib=1.7.0=pyhd8ed1ab_0
+  - tenacity=8.2.3=pyhd8ed1ab_0
+  - tensorboard=2.12.3=pyhd8ed1ab_0
+  - tensorboard-data-server=0.7.0=py310h75e40e8_1
+  - tensorflow=2.12.1=cuda112py310h457873b_1
+  - tensorflow-base=2.12.1=cuda112py310h1d65785_1
+  - tensorflow-estimator=2.12.1=cuda112py310ha5e6de5_1
+  - termcolor=2.3.0=pyhd8ed1ab_0
+  - terminado=0.18.0=pyh0d859eb_0
+  - text-unidecode=1.3=pyhd8ed1ab_1
+  - textwrap3=0.9.2=py_0
+  - thinc=8.2.1=py310hcb52e73_0
+  - threadpoolctl=3.2.0=pyha21a80b_0
+  - thrift=0.19.0=py310hc6cd4ac_1
+  - thrift_sasl=0.4.3=pyhd8ed1ab_2
+  - tifffile=2023.12.9=pyhd8ed1ab_0
+  - tiktoken=0.5.2=py310h47bb294_0
+  - timm=0.9.12=pyhd8ed1ab_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tokenizers=0.13.3=py310he1f1126_0
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tomlkit=0.12.3=pyha770c72_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchmetrics=1.0.3=pyhd8ed1ab_0
+  - torchvision=0.15.2=cuda112py310h0801bf5_1
+  - tornado=6.3.3=py310h2372a71_1
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - traitlets=5.14.0=pyhd8ed1ab_0
+  - transformers=4.31.0=pyhd8ed1ab_0
+  - truststore=0.8.0=pyhd8ed1ab_0
+  - typer=0.9.0=pyhd8ed1ab_0
+  - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0
+  - types-pytz=2023.3.1.1=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - typing_inspect=0.9.0=pyhd8ed1ab_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - typish=1.9.3=pyhd8ed1ab_0
+  - tzdata=2023c=h71feb2d_0
+  - ucx=1.14.1=h64cca9d_5
+  - ujson=5.9.0=py310hc6cd4ac_0
+  - unicodedata2=15.1.0=py310h2372a71_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - urllib3=1.26.18=pyhd8ed1ab_0
+  - wandb=0.16.1=pyhd8ed1ab_0
+  - wasabi=1.1.2=py310hff52083_0
+  - wcwidth=0.2.12=pyhd8ed1ab_0
+  - weasel=0.3.4=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.7.0=pyhd8ed1ab_0
+  - werkzeug=3.0.1=pyhd8ed1ab_0
+  - whatthepatch=1.0.5=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - widgetsnbextension=4.0.9=pyhd8ed1ab_0
+  - window-ops=0.0.14=pyhd8ed1ab_0
+  - wrapt=1.16.0=py310h2372a71_0
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xgboost=1.7.6=cuda118_py310h9840055_6
+  - xkeyboard-config=2.40=hd590300_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.7=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xxhash=0.8.2=hd590300_0
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yaml-cpp=0.8.0=h59595ed_0
+  - yapf=0.40.1=pyhd8ed1ab_0
+  - yarl=1.9.3=py310h2372a71_0
+  - zeromq=4.3.5=h59595ed_0
+  - zfp=1.0.0=h59595ed_4
+  - zict=3.0.0=pyhd8ed1ab_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zlib-ng=2.0.7=h0b41bf4_0
+  - zstandard=0.22.0=py310h1275a96_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - annotated-types==0.6.0
+      - greenlet==3.0.1
+      - jupyter-scheduler==2.4.0
+      - krb5==0.5.1
+      - peft==0.8.2
+      - pydantic==2.5.1
+      - pydantic-core==2.14.3
+      - sqlalchemy==1.4.50
+prefix: /opt/conda

wandb/run-20240208_184521-kk66dbgv/files/config.yaml

@@ -0,0 +1,614 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.13
+    cli_version: 0.16.1
+    framework: huggingface
+    huggingface_version: 4.31.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1707417921.176525
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      3:
+      - 7
+      - 23
+      4: 3.10.13
+      5: 0.16.1
+      6: 4.31.0
+      8:
+      - 5
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+vocab_size:
+  desc: null
+  value: 102400
+max_position_embeddings:
+  desc: null
+  value: 4096
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 11008
+num_hidden_layers:
+  desc: null
+  value: 30
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 32
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_scaling:
+  desc: null
+  value: null
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 100000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 100015
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+transformers_version:
+  desc: null
+  value: 4.31.0
+attention_bias:
+  desc: null
+  value: false
+model_type:
+  desc: null
+  value: llama
+rope_theta:
+  desc: null
+  value: 10000.0
+output_dir:
+  desc: null
+  value: output-7b-26k-lora
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_device_eval_batch_size:
+  desc: null
+  value: 1
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 2.0e-05
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 2
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 10
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: output-7b-26k-lora/runs/Feb08_18-45-12_default
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_steps:
+  desc: null
+  value: 100
+save_total_limit:
+  desc: null
+  value: 100
+save_safetensors:
+  desc: null
+  value: false
+save_on_each_node:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: None
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: None
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: None
+tpu_num_cores:
+  desc: null
+  value: None
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: '[]'
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: None
+dataloader_num_workers:
+  desc: null
+  value: 4
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: output-7b-26k-lora
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: None
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: None
+greater_is_better:
+  desc: null
+  value: None
+ignore_data_skip:
+  desc: null
+  value: false
+sharded_ddp:
+  desc: null
+  value: '[]'
+fsdp:
+  desc: null
+  value: '[]'
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value: '{''fsdp_min_num_params'': 0, ''xla'': False, ''xla_fsdp_grad_ckpt'': False}'
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: None
+deepspeed:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: None
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value: '[''tensorboard'', ''wandb'']'
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+ddp_broadcast_buffers:
+  desc: null
+  value: None
+dataloader_pin_memory:
+  desc: null
+  value: true
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: None
+hub_model_id:
+  desc: null
+  value: zzz99/output-7b-26k-lora
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: None
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: None
+torch_compile_mode:
+  desc: null
+  value: None
+xpu_backend:
+  desc: null
+  value: None
+cache_dir:
+  desc: null
+  value: None
+model_max_length:
+  desc: null
+  value: 960
+gradient_checkpointing_kwargs:
+  desc: null
+  value: '{''use_reentrant'': False}'
+train_batch_size:
+  desc: null
+  value: 4
+eval_batch_size:
+  desc: null
+  value: 1

wandb/run-20240208_184521-kk66dbgv/files/output.log

@@ -0,0 +1,21 @@
+  0%|                                                                                         | 0/2 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+Traceback (most recent call last):
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py", line 236, in <module>
+    train()
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py", line 223, in train
+    trainer.train()
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+    return inner_training_loop(
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2665, in training_step
+    self.accelerator.backward(loss)
+  File "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py", line 1853, in backward
+    loss.backward(**kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20240208_184521-kk66dbgv/files/requirements.txt

@@ -0,0 +1,419 @@
+absl-py==2.0.0
+accelerate==0.21.0
+aiobotocore==2.7.0
+aiohttp==3.9.1
+aioitertools==0.11.0
+aiosignal==1.3.1
+aiosqlite==0.19.0
+altair==5.2.0
+amazon-codewhisperer-jupyterlab-ext==2.0.1
+amazon-sagemaker-jupyter-scheduler==3.0.5
+annotated-types==0.6.0
+ansi2html==0.0.0
+ansiwrap==0.8.4
+antlr4-python3-runtime==4.9.3
+anyio==3.7.1
+appdirs==1.4.4
+archspec==0.2.2
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+astroid==3.0.1
+asttokens==2.4.1
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+autogluon.common==0.8.2
+autogluon.core==0.8.2
+autogluon.features==0.8.2
+autogluon.multimodal==0.8.2
+autogluon.tabular==0.8.2
+autogluon.timeseries==0.8.2
+autogluon==0.8.2
+autopep8==2.0.4
+autovizwidget==0.21.0
+aws-embedded-metrics==3.2.0
+aws-glue-sessions==1.0.4
+babel==2.13.1
+beautifulsoup4==4.12.2
+binaryornot==0.4.4
+black==23.11.0
+bleach==6.1.0
+blinker==1.7.0
+blis==0.7.10
+boltons==23.0.0
+boto3==1.28.64
+botocore==1.31.64
+brotli==1.0.9
+cached-property==1.5.2
+cachetools==5.3.2
+catalogue==2.0.10
+catboost==1.2.2
+certifi==2023.11.17
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==2.2.1
+colorama==0.4.6
+comm==0.1.4
+conda-libmamba-solver==23.11.1
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.11.0
+confection==0.1.4
+contextlib2==21.6.0
+contourpy==1.2.0
+cookiecutter==2.5.0
+croniter==1.4.1
+cryptography==41.0.7
+cycler==0.12.1
+cymem==2.0.8
+cytoolz==0.12.2
+dash==2.14.2
+dask==2023.12.0
+dataclasses-json==0.6.3
+dataclasses==0.8
+datasets==2.14.7
+debugpy==1.8.0
+decorator==5.1.1
+deepmerge==1.1.0
+defusedxml==0.7.1
+dill==0.3.7
+distributed==2023.12.0
+distro==1.8.0
+docker-pycreds==0.4.0
+docstring-to-markdown==0.13
+entrypoints==0.4
+et-xmlfile==1.1.0
+evaluate==0.4.1
+exceptiongroup==1.2.0
+executing==2.0.1
+faiss==1.7.4
+fastai==2.7.13
+fastcore==1.5.29
+fastdownload==0.0.7
+fastjsonschema==2.19.0
+fastprogress==1.0.3
+filelock==3.13.1
+flake8==6.1.0
+flask==3.0.0
+flatbuffers==23.5.26
+fonttools==4.46.0
+fqdn==1.5.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+future==0.18.3
+gast==0.4.0
+gdown==4.7.1
+gitdb==4.0.11
+gitpython==3.1.40
+gluonts==0.13.7
+gmpy2==2.1.2
+google-api-core==2.15.0
+google-auth-oauthlib==1.0.0
+google-auth==2.25.2
+google-pasta==0.2.0
+googleapis-common-protos==1.62.0
+graphviz==0.20.1
+greenlet==3.0.1
+grpcio==1.54.3
+gssapi==1.8.3
+h5py==3.10.0
+hdijupyterutils==0.21.0
+huggingface-hub==0.19.4
+idna==3.6
+imagecodecs==2023.8.12
+imageio==2.31.5
+importlib-metadata==6.10.0
+importlib-resources==6.1.1
+iniconfig==2.0.0
+ipykernel==6.26.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+isort==5.12.0
+itsdangerous==2.1.2
+jax==0.4.20
+jaxlib==0.4.14
+jedi==0.19.1
+jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpath-ng==1.6.0
+jsonpointer==2.4
+jsonschema==4.17.3
+jupyter-ai-magics==2.6.0
+jupyter-ai==2.6.0
+jupyter-client==8.6.0
+jupyter-console==6.6.3
+jupyter-core==5.5.0
+jupyter-dash==0.4.2
+jupyter-events==0.6.3
+jupyter-lsp==2.2.1
+jupyter-scheduler==2.4.0
+jupyter-server-mathjax==0.2.6
+jupyter-server-proxy==4.1.0
+jupyter-server-terminals==0.5.0
+jupyter-server==2.10.0
+jupyter==1.0.0
+jupyterlab-git==0.50.0
+jupyterlab-lsp==5.0.1
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.24.0
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+keras-preprocessing==1.1.2
+keras==2.12.0
+kiwisolver==1.4.5
+krb5==0.5.1
+langchain==0.0.318
+langcodes==3.3.0
+langsmith==0.0.69
+libmambapy==1.5.3
+lightgbm==3.3.5
+lightning-utilities==0.10.0
+llvmlite==0.41.1
+locket==1.0.0
+markdown-it-py==3.0.0
+markdown==3.5.1
+markupsafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mccabe==0.7.0
+mdurl==0.1.0
+menuinst==2.0.1
+mistune==3.0.2
+ml-dtypes==0.3.1
+mlforecast==0.7.3
+mock==5.1.0
+model-index==0.1.11
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+munkres==1.1.4
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.12.0
+nbdime==4.0.1
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nlpaug==1.1.11
+nltk==3.8.1
+nose==1.3.7
+notebook-shim==0.2.3
+notebook==7.0.6
+nptyping==2.4.1
+numba==0.58.1
+numexpr==2.8.7
+numpy==1.26.2
+oauthlib==3.2.2
+omegaconf==2.2.3
+openai==0.28.1
+openapi-schema-pydantic==1.2.4
+openmim==0.3.7
+openpyxl==3.1.2
+opt-einsum==3.3.0
+ordered-set==4.1.0
+overrides==7.4.0
+packaging==23.2
+pandas-stubs==2.1.1.230928
+pandas==2.1.4
+pandocfilters==1.5.0
+papermill==2.4.0
+parso==0.8.3
+partd==1.4.1
+pathos==0.3.1
+pathspec==0.12.1
+pathtools==0.1.2
+pathy==0.10.2
+patsy==0.5.4
+peft==0.8.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.5.0
+pip==23.3.1
+pkgutil-resolve-name==1.3.10
+platformdirs==4.1.0
+plotly==5.18.0
+pluggy==1.3.0
+ply==3.11
+pox==0.3.3
+ppft==1.7.6.7
+preshed==3.0.9
+prometheus-client==0.19.0
+prompt-toolkit==3.0.41
+protobuf==4.21.12
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pure-sasl==0.6.2
+pyarrow-hotfix==0.6
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pycodestyle==2.11.1
+pycosat==0.6.6
+pycparser==2.21
+pydantic-core==2.14.3
+pydantic==1.10.13
+pydocstyle==6.3.0
+pyflakes==3.1.0
+pygments==2.17.2
+pyhive==0.7.0
+pyjwt==2.8.0
+pylint==3.0.3
+pyopenssl==23.3.0
+pyparsing==3.1.1
+pyqt5-sip==12.12.2
+pyqt5==5.15.9
+pyrsistent==0.20.0
+pysocks==1.7.1
+pyspnego==0.9.1
+pytesseract==0.3.10
+pytest-subtests==0.11.0
+pytest==7.4.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.9.0
+python-slugify==8.0.1
+pytoolconfig==1.2.5
+pytorch-lightning==2.0.9
+pytorch-metric-learning==1.7.3
+pytz==2023.3
+pyu2f==0.1.5
+pywavelets==1.4.1
+pyyaml==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+qtpy==2.4.1
+regex==2023.10.3
+requests-kerberos==0.14.0
+requests-oauthlib==1.3.1
+requests==2.31.0
+responses==0.18.0
+retrying==1.3.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rope==1.11.0
+rsa==4.9
+ruamel.yaml.clib==0.2.7
+ruamel.yaml==0.18.5
+s3transfer==0.7.0
+sacremoses==0.0.53
+safetensors==0.3.3
+sagemaker-headless-execution-driver==0.0.12
+sagemaker-jupyterlab-emr-extension==0.1.9
+sagemaker-jupyterlab-extension-common==0.1.9
+sagemaker-jupyterlab-extension==0.2.0
+sagemaker-kernel-wrapper==0.0.2
+sagemaker-studio-analytics-extension==0.0.21
+sagemaker-studio-sparkmagic-lib==0.1.4
+sagemaker==2.198.0
+sasl==0.3.1
+schema==0.7.5
+scikit-image==0.19.3
+scikit-learn==1.3.2
+scipy==1.11.4
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==1.38.0
+seqeval==1.2.2
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+simpervisor==1.0.0
+sip==6.7.12
+six==1.16.0
+smart-open==5.2.1
+smdebug-rulesconfig==1.0.1
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sparkmagic==0.21.0
+sqlalchemy==1.4.49
+srsly==2.4.8
+stack-data==0.6.2
+statsforecast==1.4.0
+statsmodels==0.14.0
+supervisor==4.2.5
+sympy==1.12
+tabulate==0.9.0
+tblib==1.7.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.0
+tensorboard==2.12.3
+tensorflow-estimator==2.12.0
+tensorflow==2.12.1
+termcolor==2.3.0
+terminado==0.18.0
+text-unidecode==1.3
+textwrap3==0.9.2
+thinc==8.2.1
+threadpoolctl==3.2.0
+thrift-sasl==0.4.3
+thrift==0.19.0
+tifffile==2023.12.9
+tiktoken==0.5.2
+timm==0.9.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.12.3
+toolz==0.12.0
+torch==2.0.0.post200
+torchmetrics==1.0.3
+torchvision==0.15.2a0+072ec57
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.31.0
+truststore==0.8.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+types-pytz==2023.3.1.1
+typing-extensions==4.5.0
+typing-inspect==0.9.0
+typing-utils==0.1.0
+typish==1.9.3
+tzdata==2023.3
+ujson==5.9.0
+unicodedata2==15.1.0
+uri-template==1.3.0
+urllib3==1.26.18
+wandb==0.16.1
+wasabi==1.1.2
+wcwidth==0.2.12
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.1
+whatthepatch==1.0.5
+wheel==0.42.0
+widgetsnbextension==4.0.9
+window-ops==0.0.14
+wrapt==1.16.0
+xgboost==1.7.6
+xxhash==3.4.1
+yapf==0.40.1
+yarl==1.9.3
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.22.0

wandb/run-20240208_184521-kk66dbgv/files/wandb-metadata.json

@@ -0,0 +1,1029 @@
+{
+    "os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
+    "python": "3.10.13",
+    "heartbeatAt": "2024-02-08T18:45:21.624647",
+    "startedAt": "2024-02-08T18:45:21.172989",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py",
+    "codePathLocal": null,
+    "host": "default",
+    "username": "sagemaker-user",
+    "executable": "/opt/conda/bin/python3",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 3165.433677083333,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3299.529,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.574,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.907,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.263,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.897,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.385,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.81,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.625,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.926,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.158,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.475,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.688,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3295.206,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3294.95,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.334,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.722,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.096,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.885,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.66,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.613,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.244,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3241.28,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.967,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.457,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.049,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.552,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.807,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3242.538,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.129,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3263.29,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.421,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.256,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.723,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.38,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.22,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.243,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3259.228,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.656,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.572,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.246,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.507,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.177,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.762,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.244,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.764,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.71,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.323,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.972,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2825.298,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.031,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.524,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.753,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.281,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.549,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.256,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.719,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.975,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.721,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.6,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.408,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.691,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.817,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3044.848,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.416,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.089,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.351,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2807.753,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2853.085,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.456,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.145,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.532,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.121,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.716,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2964.818,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.325,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3053.968,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.575,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3034.933,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3046.955,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3017.189,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3052.512,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3049.645,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3056.957,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3063.442,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3026.186,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.995,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3058.868,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.978,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2639.982,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3043.356,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.312,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3024.784,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3309.767,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3044.167,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3074.821,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2744.486,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2948.546,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.131,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3260.141,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3264.163,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.133,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3260.992,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.601,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3266.096,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.245,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.423,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3262.508,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3270.751,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3265.57,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3268.221,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3262.709,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3262.206,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3270.565,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.66,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3271.159,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3269.543,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2891.532,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.121,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3267.57,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3273.911,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3271.579,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3271.885,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3269.181,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.12,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3272.274,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.966,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.849,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.555,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.44,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.027,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.417,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.561,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.684,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.308,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.07,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.982,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.738,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.558,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.74,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.099,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.072,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.608,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3293.695,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.228,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.509,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.722,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.9,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.551,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.029,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.307,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.752,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.526,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.18,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.048,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.113,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.319,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.493,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.269,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.472,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.484,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.416,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.596,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.52,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.897,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.216,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.001,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.316,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2995.097,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2690.969,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.22,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3008.014,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.622,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2987.966,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3021.177,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.724,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2997.024,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3036.103,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2998.071,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.959,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3043.183,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.567,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.171,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2961.029,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.873,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3037.985,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3009.778,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3032.565,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3272.763,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3109.523,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.902,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3283.894,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 32.0,
+            "used": 0.012481689453125
+        }
+    },
+    "gpu": "NVIDIA A10G",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        }
+    ],
+    "memory": {
+        "total": 747.9597625732422
+    }
+}

wandb/run-20240208_184521-kk66dbgv/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 17}}

wandb/run-20240208_184521-kk66dbgv/logs/debug-internal.log

@@ -0,0 +1,194 @@
+2024-02-08 18:45:21,177 INFO    StreamThr :789 [internal.py:wandb_internal():86] W&B internal server running at pid: 789, started at: 2024-02-08 18:45:21.176939
+2024-02-08 18:45:21,181 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: status
+2024-02-08 18:45:21,182 INFO    WriterThread:789 [datastore.py:open_for_write():85] open: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/run-kk66dbgv.wandb
+2024-02-08 18:45:21,183 DEBUG   SenderThread:789 [sender.py:send():382] send: header
+2024-02-08 18:45:21,183 DEBUG   SenderThread:789 [sender.py:send():382] send: run
+2024-02-08 18:45:21,486 INFO    SenderThread:789 [dir_watcher.py:__init__():211] watching files in: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files
+2024-02-08 18:45:21,486 INFO    SenderThread:789 [sender.py:_start_run_threads():1136] run started: kk66dbgv with start time 1707417921.176525
+2024-02-08 18:45:21,490 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: check_version
+2024-02-08 18:45:21,491 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: check_version
+2024-02-08 18:45:21,534 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: run_start
+2024-02-08 18:45:21,565 DEBUG   HandlerThread:789 [system_info.py:__init__():32] System info init
+2024-02-08 18:45:21,565 DEBUG   HandlerThread:789 [system_info.py:__init__():47] System info init done
+2024-02-08 18:45:21,565 INFO    HandlerThread:789 [system_monitor.py:start():194] Starting system monitor
+2024-02-08 18:45:21,565 INFO    SystemMonitor:789 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-02-08 18:45:21,566 INFO    HandlerThread:789 [system_monitor.py:probe():214] Collecting system info
+2024-02-08 18:45:21,566 INFO    SystemMonitor:789 [interfaces.py:start():190] Started cpu monitoring
+2024-02-08 18:45:21,567 INFO    SystemMonitor:789 [interfaces.py:start():190] Started disk monitoring
+2024-02-08 18:45:21,568 INFO    SystemMonitor:789 [interfaces.py:start():190] Started gpu monitoring
+2024-02-08 18:45:21,568 INFO    SystemMonitor:789 [interfaces.py:start():190] Started memory monitoring
+2024-02-08 18:45:21,569 INFO    SystemMonitor:789 [interfaces.py:start():190] Started network monitoring
+2024-02-08 18:45:21,624 DEBUG   HandlerThread:789 [system_info.py:probe():196] Probing system
+2024-02-08 18:45:21,626 DEBUG   HandlerThread:789 [gitlib.py:_init_repo():56] git repository is invalid
+2024-02-08 18:45:21,626 DEBUG   HandlerThread:789 [system_info.py:probe():244] Probing system done
+2024-02-08 18:45:21,626 DEBUG   HandlerThread:789 [system_monitor.py:probe():223] {'os': 'Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35', 'python': '3.10.13', 'heartbeatAt': '2024-02-08T18:45:21.624647', 'startedAt': '2024-02-08T18:45:21.172989', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py', 'codePathLocal': None, 'host': 'default', 'username': 'sagemaker-user', 'executable': '/opt/conda/bin/python3', 'cpu_count': 96, 'cpu_count_logical': 192, 'cpu_freq': {'current': 3165.433677083333, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 3299.529, 'min': 0.0, 'max': 0.0}, {'current': 3299.574, 'min': 0.0, 'max': 0.0}, {'current': 3299.907, 'min': 0.0, 'max': 0.0}, {'current': 3300.263, 'min': 0.0, 'max': 0.0}, {'current': 3300.897, 'min': 0.0, 'max': 0.0}, {'current': 3300.385, 'min': 0.0, 'max': 0.0}, {'current': 3298.81, 'min': 0.0, 'max': 0.0}, {'current': 3299.625, 'min': 0.0, 'max': 0.0}, {'current': 3299.926, 'min': 0.0, 'max': 0.0}, {'current': 3300.158, 'min': 0.0, 'max': 0.0}, {'current': 3300.475, 'min': 0.0, 'max': 0.0}, {'current': 3300.688, 'min': 0.0, 'max': 0.0}, {'current': 3295.206, 'min': 0.0, 'max': 0.0}, {'current': 3294.95, 'min': 0.0, 'max': 0.0}, {'current': 3296.334, 'min': 0.0, 'max': 0.0}, {'current': 3297.722, 'min': 0.0, 'max': 0.0}, {'current': 3296.096, 'min': 0.0, 'max': 0.0}, {'current': 3298.885, 'min': 0.0, 'max': 0.0}, {'current': 3297.66, 'min': 0.0, 'max': 0.0}, {'current': 3297.613, 'min': 0.0, 'max': 0.0}, {'current': 3300.244, 'min': 0.0, 'max': 0.0}, {'current': 3241.28, 'min': 0.0, 'max': 0.0}, {'current': 3298.967, 'min': 0.0, 'max': 0.0}, {'current': 3298.457, 'min': 0.0, 'max': 0.0}, {'current': 3298.049, 'min': 0.0, 'max': 0.0}, {'current': 3299.552, 'min': 0.0, 'max': 0.0}, {'current': 3299.807, 'min': 0.0, 'max': 0.0}, {'current': 3242.538, 'min': 0.0, 'max': 0.0}, {'current': 3299.129, 'min': 0.0, 'max': 0.0}, {'current': 3263.29, 'min': 0.0, 'max': 0.0}, {'current': 3298.421, 'min': 0.0, 'max': 0.0}, {'current': 3299.256, 'min': 0.0, 'max': 0.0}, {'current': 3298.723, 'min': 0.0, 'max': 0.0}, {'current': 3299.38, 'min': 0.0, 'max': 0.0}, {'current': 3299.22, 'min': 0.0, 'max': 0.0}, {'current': 3298.243, 'min': 0.0, 'max': 0.0}, {'current': 3259.228, 'min': 0.0, 'max': 0.0}, {'current': 3297.656, 'min': 0.0, 'max': 0.0}, {'current': 3299.572, 'min': 0.0, 'max': 0.0}, {'current': 3299.246, 'min': 0.0, 'max': 0.0}, {'current': 3299.507, 'min': 0.0, 'max': 0.0}, {'current': 3298.177, 'min': 0.0, 'max': 0.0}, {'current': 3299.762, 'min': 0.0, 'max': 0.0}, {'current': 3300.244, 'min': 0.0, 'max': 0.0}, {'current': 3299.764, 'min': 0.0, 'max': 0.0}, {'current': 3299.71, 'min': 0.0, 'max': 0.0}, {'current': 3299.323, 'min': 0.0, 'max': 0.0}, {'current': 3298.972, 'min': 0.0, 'max': 0.0}, {'current': 2825.298, 'min': 0.0, 'max': 0.0}, {'current': 3300.031, 'min': 0.0, 'max': 0.0}, {'current': 3299.524, 'min': 0.0, 'max': 0.0}, {'current': 3300.753, 'min': 0.0, 'max': 0.0}, {'current': 3300.281, 'min': 0.0, 'max': 0.0}, {'current': 3300.549, 'min': 0.0, 'max': 0.0}, {'current': 3299.256, 'min': 0.0, 'max': 0.0}, {'current': 3300.719, 'min': 0.0, 'max': 0.0}, {'current': 3299.975, 'min': 0.0, 'max': 0.0}, {'current': 3300.721, 'min': 0.0, 'max': 0.0}, {'current': 3300.6, 'min': 0.0, 'max': 0.0}, {'current': 3300.408, 'min': 0.0, 'max': 0.0}, {'current': 3299.691, 'min': 0.0, 'max': 0.0}, {'current': 3299.817, 'min': 0.0, 'max': 0.0}, {'current': 3044.848, 'min': 0.0, 'max': 0.0}, {'current': 3299.416, 'min': 0.0, 'max': 0.0}, {'current': 3300.089, 'min': 0.0, 'max': 0.0}, {'current': 3299.351, 'min': 0.0, 'max': 0.0}, {'current': 2807.753, 'min': 0.0, 'max': 0.0}, {'current': 2853.085, 'min': 0.0, 'max': 0.0}, {'current': 3299.456, 'min': 0.0, 'max': 0.0}, {'current': 3300.145, 'min': 0.0, 'max': 0.0}, {'current': 3299.532, 'min': 0.0, 'max': 0.0}, {'current': 3300.121, 'min': 0.0, 'max': 0.0}, {'current': 3298.716, 'min': 0.0, 'max': 0.0}, {'current': 2964.818, 'min': 0.0, 'max': 0.0}, {'current': 3299.325, 'min': 0.0, 'max': 0.0}, {'current': 3053.968, 'min': 0.0, 'max': 0.0}, {'current': 3027.575, 'min': 0.0, 'max': 0.0}, {'current': 3034.933, 'min': 0.0, 'max': 0.0}, {'current': 3046.955, 'min': 0.0, 'max': 0.0}, {'current': 3017.189, 'min': 0.0, 'max': 0.0}, {'current': 3052.512, 'min': 0.0, 'max': 0.0}, {'current': 3049.645, 'min': 0.0, 'max': 0.0}, {'current': 3056.957, 'min': 0.0, 'max': 0.0}, {'current': 3063.442, 'min': 0.0, 'max': 0.0}, {'current': 3026.186, 'min': 0.0, 'max': 0.0}, {'current': 3059.995, 'min': 0.0, 'max': 0.0}, {'current': 3058.868, 'min': 0.0, 'max': 0.0}, {'current': 3059.978, 'min': 0.0, 'max': 0.0}, {'current': 2639.982, 'min': 0.0, 'max': 0.0}, {'current': 3043.356, 'min': 0.0, 'max': 0.0}, {'current': 3032.312, 'min': 0.0, 'max': 0.0}, {'current': 3024.784, 'min': 0.0, 'max': 0.0}, {'current': 3309.767, 'min': 0.0, 'max': 0.0}, {'current': 3044.167, 'min': 0.0, 'max': 0.0}, {'current': 3074.821, 'min': 0.0, 'max': 0.0}, {'current': 2744.486, 'min': 0.0, 'max': 0.0}, {'current': 2948.546, 'min': 0.0, 'max': 0.0}, {'current': 3265.131, 'min': 0.0, 'max': 0.0}, {'current': 3260.141, 'min': 0.0, 'max': 0.0}, {'current': 3264.163, 'min': 0.0, 'max': 0.0}, {'current': 3299.133, 'min': 0.0, 'max': 0.0}, {'current': 3260.992, 'min': 0.0, 'max': 0.0}, {'current': 3299.601, 'min': 0.0, 'max': 0.0}, {'current': 3266.096, 'min': 0.0, 'max': 0.0}, {'current': 3299.245, 'min': 0.0, 'max': 0.0}, {'current': 3298.423, 'min': 0.0, 'max': 0.0}, {'current': 3262.508, 'min': 0.0, 'max': 0.0}, {'current': 3270.751, 'min': 0.0, 'max': 0.0}, {'current': 3265.57, 'min': 0.0, 'max': 0.0}, {'current': 3268.221, 'min': 0.0, 'max': 0.0}, {'current': 3262.709, 'min': 0.0, 'max': 0.0}, {'current': 3262.206, 'min': 0.0, 'max': 0.0}, {'current': 3270.565, 'min': 0.0, 'max': 0.0}, {'current': 3298.66, 'min': 0.0, 'max': 0.0}, {'current': 3271.159, 'min': 0.0, 'max': 0.0}, {'current': 3269.543, 'min': 0.0, 'max': 0.0}, {'current': 2891.532, 'min': 0.0, 'max': 0.0}, {'current': 3299.121, 'min': 0.0, 'max': 0.0}, {'current': 3267.57, 'min': 0.0, 'max': 0.0}, {'current': 3273.911, 'min': 0.0, 'max': 0.0}, {'current': 3271.579, 'min': 0.0, 'max': 0.0}, {'current': 3271.885, 'min': 0.0, 'max': 0.0}, {'current': 3269.181, 'min': 0.0, 'max': 0.0}, {'current': 3299.12, 'min': 0.0, 'max': 0.0}, {'current': 3272.274, 'min': 0.0, 'max': 0.0}, {'current': 3298.966, 'min': 0.0, 'max': 0.0}, {'current': 3298.849, 'min': 0.0, 'max': 0.0}, {'current': 3298.555, 'min': 0.0, 'max': 0.0}, {'current': 3298.44, 'min': 0.0, 'max': 0.0}, {'current': 3299.027, 'min': 0.0, 'max': 0.0}, {'current': 3299.417, 'min': 0.0, 'max': 0.0}, {'current': 3298.561, 'min': 0.0, 'max': 0.0}, {'current': 3298.684, 'min': 0.0, 'max': 0.0}, {'current': 3298.308, 'min': 0.0, 'max': 0.0}, {'current': 3299.07, 'min': 0.0, 'max': 0.0}, {'current': 3297.982, 'min': 0.0, 'max': 0.0}, {'current': 3298.738, 'min': 0.0, 'max': 0.0}, {'current': 3297.558, 'min': 0.0, 'max': 0.0}, {'current': 3297.74, 'min': 0.0, 'max': 0.0}, {'current': 3299.099, 'min': 0.0, 'max': 0.0}, {'current': 3299.072, 'min': 0.0, 'max': 0.0}, {'current': 3298.608, 'min': 0.0, 'max': 0.0}, {'current': 3299.045, 'min': 0.0, 'max': 0.0}, {'current': 3293.695, 'min': 0.0, 'max': 0.0}, {'current': 3299.228, 'min': 0.0, 'max': 0.0}, {'current': 3299.509, 'min': 0.0, 'max': 0.0}, {'current': 3298.722, 'min': 0.0, 'max': 0.0}, {'current': 3299.9, 'min': 0.0, 'max': 0.0}, {'current': 3299.551, 'min': 0.0, 'max': 0.0}, {'current': 3299.029, 'min': 0.0, 'max': 0.0}, {'current': 3299.307, 'min': 0.0, 'max': 0.0}, {'current': 3298.752, 'min': 0.0, 'max': 0.0}, {'current': 3299.526, 'min': 0.0, 'max': 0.0}, {'current': 3299.18, 'min': 0.0, 'max': 0.0}, {'current': 3299.048, 'min': 0.0, 'max': 0.0}, {'current': 3299.113, 'min': 0.0, 'max': 0.0}, {'current': 3299.319, 'min': 0.0, 'max': 0.0}, {'current': 3299.493, 'min': 0.0, 'max': 0.0}, {'current': 3299.269, 'min': 0.0, 'max': 0.0}, {'current': 3299.472, 'min': 0.0, 'max': 0.0}, {'current': 3299.484, 'min': 0.0, 'max': 0.0}, {'current': 3299.416, 'min': 0.0, 'max': 0.0}, {'current': 3299.596, 'min': 0.0, 'max': 0.0}, {'current': 3299.52, 'min': 0.0, 'max': 0.0}, {'current': 3298.897, 'min': 0.0, 'max': 0.0}, {'current': 3299.216, 'min': 0.0, 'max': 0.0}, {'current': 3299.001, 'min': 0.0, 'max': 0.0}, {'current': 3300.316, 'min': 0.0, 'max': 0.0}, {'current': 2995.097, 'min': 0.0, 'max': 0.0}, {'current': 2690.969, 'min': 0.0, 'max': 0.0}, {'current': 3300.22, 'min': 0.0, 'max': 0.0}, {'current': 3008.014, 'min': 0.0, 'max': 0.0}, {'current': 3299.622, 'min': 0.0, 'max': 0.0}, {'current': 2987.966, 'min': 0.0, 'max': 0.0}, {'current': 3021.177, 'min': 0.0, 'max': 0.0}, {'current': 3032.724, 'min': 0.0, 'max': 0.0}, {'current': 2997.024, 'min': 0.0, 'max': 0.0}, {'current': 3036.103, 'min': 0.0, 'max': 0.0}, {'current': 2998.071, 'min': 0.0, 'max': 0.0}, {'current': 3298.959, 'min': 0.0, 'max': 0.0}, {'current': 3043.183, 'min': 0.0, 'max': 0.0}, {'current': 3299.567, 'min': 0.0, 'max': 0.0}, {'current': 3027.171, 'min': 0.0, 'max': 0.0}, {'current': 2961.029, 'min': 0.0, 'max': 0.0}, {'current': 3059.873, 'min': 0.0, 'max': 0.0}, {'current': 3037.985, 'min': 0.0, 'max': 0.0}, {'current': 3009.778, 'min': 0.0, 'max': 0.0}, {'current': 3032.565, 'min': 0.0, 'max': 0.0}, {'current': 3272.763, 'min': 0.0, 'max': 0.0}, {'current': 3109.523, 'min': 0.0, 'max': 0.0}, {'current': 3299.902, 'min': 0.0, 'max': 0.0}, {'current': 3283.894, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 32.0, 'used': 0.012481689453125}}, 'gpu': 'NVIDIA A10G', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}, {'name': 'NVIDIA A10G', 'memory_total': 24146608128}], 'memory': {'total': 747.9597625732422}}
+2024-02-08 18:45:21,626 INFO    HandlerThread:789 [system_monitor.py:probe():224] Finished collecting system info
+2024-02-08 18:45:21,626 INFO    HandlerThread:789 [system_monitor.py:probe():227] Publishing system info
+2024-02-08 18:45:21,626 DEBUG   HandlerThread:789 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2024-02-08 18:45:21,627 DEBUG   HandlerThread:789 [system_info.py:_save_pip():68] Saving pip packages done
+2024-02-08 18:45:21,627 DEBUG   HandlerThread:789 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2024-02-08 18:45:22,487 INFO    Thread-12 :789 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/conda-environment.yaml
+2024-02-08 18:45:22,487 INFO    Thread-12 :789 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/requirements.txt
+2024-02-08 18:45:35,889 DEBUG   HandlerThread:789 [system_info.py:_save_conda():87] Saving conda packages done
+2024-02-08 18:45:35,890 INFO    HandlerThread:789 [system_monitor.py:probe():229] Finished publishing system info
+2024-02-08 18:45:35,894 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:45:35,894 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:45:35,894 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:45:35,894 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: keepalive
+2024-02-08 18:45:35,895 DEBUG   SenderThread:789 [sender.py:send():382] send: files
+2024-02-08 18:45:35,895 INFO    SenderThread:789 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now
+2024-02-08 18:45:35,899 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-08 18:45:35,900 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: stop_status
+2024-02-08 18:45:35,906 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:45:36,041 DEBUG   SenderThread:789 [sender.py:send():382] send: telemetry
+2024-02-08 18:45:36,042 DEBUG   SenderThread:789 [sender.py:send():382] send: config
+2024-02-08 18:45:36,042 DEBUG   SenderThread:789 [sender.py:send():382] send: metric
+2024-02-08 18:45:36,042 DEBUG   SenderThread:789 [sender.py:send():382] send: telemetry
+2024-02-08 18:45:36,042 DEBUG   SenderThread:789 [sender.py:send():382] send: metric
+2024-02-08 18:45:36,042 WARNING SenderThread:789 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen)
+2024-02-08 18:45:36,244 INFO    wandb-upload_0:789 [upload_job.py:push():131] Uploaded file /tmp/tmph6r9wm0rwandb/nitc481h-wandb-metadata.json
+2024-02-08 18:45:36,488 INFO    Thread-12 :789 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/conda-environment.yaml
+2024-02-08 18:45:36,488 INFO    Thread-12 :789 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/wandb-metadata.json
+2024-02-08 18:45:36,488 INFO    Thread-12 :789 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/output.log
+2024-02-08 18:45:36,754 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:45:38,488 INFO    Thread-12 :789 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/output.log
+2024-02-08 18:45:38,842 DEBUG   SenderThread:789 [sender.py:send():382] send: exit
+2024-02-08 18:45:38,842 INFO    SenderThread:789 [sender.py:send_exit():589] handling exit code: 1
+2024-02-08 18:45:38,842 INFO    SenderThread:789 [sender.py:send_exit():591] handling runtime: 17
+2024-02-08 18:45:38,842 INFO    SenderThread:789 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:45:38,842 INFO    SenderThread:789 [sender.py:send_exit():597] send defer
+2024-02-08 18:45:38,843 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,843 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 0
+2024-02-08 18:45:38,843 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,843 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-02-08 18:45:38,843 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 1
+2024-02-08 18:45:38,843 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,843 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 1
+2024-02-08 18:45:38,843 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,843 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-02-08 18:45:38,843 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 2
+2024-02-08 18:45:38,843 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,843 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 2
+2024-02-08 18:45:38,843 INFO    HandlerThread:789 [system_monitor.py:finish():203] Stopping system monitor
+2024-02-08 18:45:38,844 DEBUG   SystemMonitor:789 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-02-08 18:45:38,844 INFO    HandlerThread:789 [interfaces.py:finish():202] Joined cpu monitor
+2024-02-08 18:45:38,844 DEBUG   SystemMonitor:789 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-02-08 18:45:38,845 INFO    HandlerThread:789 [interfaces.py:finish():202] Joined disk monitor
+2024-02-08 18:45:38,845 DEBUG   SystemMonitor:789 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-02-08 18:45:38,884 INFO    HandlerThread:789 [interfaces.py:finish():202] Joined gpu monitor
+2024-02-08 18:45:38,884 INFO    HandlerThread:789 [interfaces.py:finish():202] Joined memory monitor
+2024-02-08 18:45:38,884 INFO    HandlerThread:789 [interfaces.py:finish():202] Joined network monitor
+2024-02-08 18:45:38,885 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,885 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-02-08 18:45:38,885 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 3
+2024-02-08 18:45:38,885 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,885 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 3
+2024-02-08 18:45:38,885 DEBUG   SenderThread:789 [sender.py:send():382] send: stats
+2024-02-08 18:45:38,886 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,886 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-02-08 18:45:38,886 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 4
+2024-02-08 18:45:38,886 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,886 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 4
+2024-02-08 18:45:38,886 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,886 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-02-08 18:45:38,886 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 5
+2024-02-08 18:45:38,886 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,886 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 5
+2024-02-08 18:45:38,887 DEBUG   SenderThread:789 [sender.py:send():382] send: summary
+2024-02-08 18:45:38,888 INFO    SenderThread:789 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-08 18:45:38,888 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,888 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-02-08 18:45:38,888 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 6
+2024-02-08 18:45:38,888 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:38,888 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 6
+2024-02-08 18:45:38,888 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:38,888 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-02-08 18:45:38,893 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: status_report
+2024-02-08 18:45:39,019 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 7
+2024-02-08 18:45:39,019 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:39,019 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 7
+2024-02-08 18:45:39,019 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:39,020 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-02-08 18:45:39,489 INFO    Thread-12 :789 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/config.yaml
+2024-02-08 18:45:39,489 INFO    Thread-12 :789 [dir_watcher.py:_on_file_created():271] file/dir created: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/wandb-summary.json
+2024-02-08 18:45:39,842 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:45:40,054 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 8
+2024-02-08 18:45:40,054 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:45:40,054 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:40,055 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 8
+2024-02-08 18:45:40,055 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:40,055 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-02-08 18:45:40,055 INFO    SenderThread:789 [job_builder.py:build():298] Attempting to build job artifact
+2024-02-08 18:45:40,056 INFO    SenderThread:789 [job_builder.py:_get_source_type():439] no source found
+2024-02-08 18:45:40,056 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 9
+2024-02-08 18:45:40,056 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:40,056 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 9
+2024-02-08 18:45:40,056 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:40,057 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-02-08 18:45:40,057 INFO    SenderThread:789 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-02-08 18:45:40,489 INFO    Thread-12 :789 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/output.log
+2024-02-08 18:45:40,489 INFO    SenderThread:789 [dir_watcher.py:finish():388] scan: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/config.yaml config.yaml
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/requirements.txt requirements.txt
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/conda-environment.yaml conda-environment.yaml
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/wandb-metadata.json wandb-metadata.json
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/output.log output.log
+2024-02-08 18:45:40,490 INFO    SenderThread:789 [dir_watcher.py:finish():402] scan save: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/wandb-summary.json wandb-summary.json
+2024-02-08 18:45:40,493 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 10
+2024-02-08 18:45:40,493 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:40,493 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 10
+2024-02-08 18:45:40,502 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:40,502 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-02-08 18:45:40,502 INFO    SenderThread:789 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:45:40,709 INFO    wandb-upload_1:789 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/config.yaml
+2024-02-08 18:45:40,784 INFO    wandb-upload_2:789 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/conda-environment.yaml
+2024-02-08 18:45:40,825 INFO    wandb-upload_4:789 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/wandb-summary.json
+2024-02-08 18:45:40,825 INFO    wandb-upload_3:789 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/output.log
+2024-02-08 18:45:40,843 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:45:40,843 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:45:40,852 INFO    wandb-upload_0:789 [upload_job.py:push():131] Uploaded file /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/files/requirements.txt
+2024-02-08 18:45:41,053 INFO    Thread-11 (_thread_body):789 [sender.py:transition_state():617] send defer: 11
+2024-02-08 18:45:41,053 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:41,053 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 11
+2024-02-08 18:45:41,053 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:41,053 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-02-08 18:45:41,053 INFO    SenderThread:789 [file_pusher.py:join():181] waiting for file pusher
+2024-02-08 18:45:41,054 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 12
+2024-02-08 18:45:41,054 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:41,054 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 12
+2024-02-08 18:45:41,054 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:41,054 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-02-08 18:45:41,054 INFO    SenderThread:789 [file_stream.py:finish():595] file stream finish called
+2024-02-08 18:45:41,126 INFO    SenderThread:789 [file_stream.py:finish():599] file stream finish is done
+2024-02-08 18:45:41,126 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 13
+2024-02-08 18:45:41,126 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:41,126 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 13
+2024-02-08 18:45:41,127 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:41,127 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-02-08 18:45:41,127 INFO    SenderThread:789 [sender.py:transition_state():617] send defer: 14
+2024-02-08 18:45:41,127 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: defer
+2024-02-08 18:45:41,127 INFO    HandlerThread:789 [handler.py:handle_request_defer():172] handle defer: 14
+2024-02-08 18:45:41,127 DEBUG   SenderThread:789 [sender.py:send():382] send: final
+2024-02-08 18:45:41,127 DEBUG   SenderThread:789 [sender.py:send():382] send: footer
+2024-02-08 18:45:41,127 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: defer
+2024-02-08 18:45:41,127 INFO    SenderThread:789 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-02-08 18:45:41,128 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:45:41,128 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:45:41,128 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: poll_exit
+2024-02-08 18:45:41,129 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: poll_exit
+2024-02-08 18:45:41,129 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: server_info
+2024-02-08 18:45:41,129 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: get_summary
+2024-02-08 18:45:41,130 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: server_info
+2024-02-08 18:45:41,131 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: sampled_history
+2024-02-08 18:45:41,132 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-08 18:45:41,132 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: job_info
+2024-02-08 18:45:41,196 DEBUG   SenderThread:789 [sender.py:send_request():409] send_request: job_info
+2024-02-08 18:45:41,197 INFO    MainThread:789 [wandb_run.py:_footer_history_summary_info():3837] rendering history
+2024-02-08 18:45:41,197 INFO    MainThread:789 [wandb_run.py:_footer_history_summary_info():3869] rendering summary
+2024-02-08 18:45:41,197 INFO    MainThread:789 [wandb_run.py:_footer_sync_info():3796] logging synced files
+2024-02-08 18:45:41,197 DEBUG   HandlerThread:789 [handler.py:handle_request():146] handle_request: shutdown
+2024-02-08 18:45:41,197 INFO    HandlerThread:789 [handler.py:finish():866] shutting down handler
+2024-02-08 18:45:42,132 INFO    WriterThread:789 [datastore.py:close():294] close: /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/run-kk66dbgv.wandb
+2024-02-08 18:45:42,197 INFO    SenderThread:789 [sender.py:finish():1548] shutting down sender
+2024-02-08 18:45:42,197 INFO    SenderThread:789 [file_pusher.py:finish():175] shutting down file pusher
+2024-02-08 18:45:42,197 INFO    SenderThread:789 [file_pusher.py:join():181] waiting for file pusher

wandb/run-20240208_184521-kk66dbgv/logs/debug.log

@@ -0,0 +1,30 @@
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Configure stats pid to 608
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/.config/wandb/settings
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Loading settings from /home/sagemaker-user/output-7b-26k-lora/wandb/settings
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-08 18:45:21,174 WARNING MainThread:608 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/home/sagemaker-user/lora_finetuning_push_to_hub_save_local_latest.py', 'program': '/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py'}
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:_log_setup():524] Logging user logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/logs/debug.log
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:_log_setup():525] Logging internal logs to /home/sagemaker-user/output-7b-26k-lora/wandb/run-20240208_184521-kk66dbgv/logs/debug-internal.log
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:init():564] calling init triggers
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+config: {}
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:init():614] starting backend
+2024-02-08 18:45:21,174 INFO    MainThread:608 [wandb_init.py:init():618] setting up manager
+2024-02-08 18:45:21,175 INFO    MainThread:608 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-08 18:45:21,176 INFO    MainThread:608 [wandb_init.py:init():624] backend started and connected
+2024-02-08 18:45:21,178 INFO    MainThread:608 [wandb_init.py:init():716] updated telemetry
+2024-02-08 18:45:21,178 INFO    MainThread:608 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-08 18:45:21,490 INFO    MainThread:608 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-08 18:45:21,530 INFO    MainThread:608 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.3 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-02-08 18:45:21,530 INFO    MainThread:608 [wandb_init.py:init():800] starting run threads in backend
+2024-02-08 18:45:35,900 INFO    MainThread:608 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-08 18:45:35,900 INFO    MainThread:608 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-08 18:45:35,900 INFO    MainThread:608 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-08 18:45:35,900 INFO    MainThread:608 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-08 18:45:35,901 INFO    MainThread:608 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-08 18:45:35,903 INFO    MainThread:608 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 102400, 'max_position_embeddings': 4096, 'hidden_size': 4096, 'intermediate_size': 11008, 'num_hidden_layers': 30, 'num_attention_heads': 32, 'num_key_value_heads': 32, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_scaling': None, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 100000, 'pad_token_id': 0, 'eos_token_id': 100015, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'deepseek-ai/deepseek-coder-7b-instruct-v1.5', 'transformers_version': '4.31.0', 'attention_bias': False, 'model_type': 'llama', 'rope_theta': 10000.0, 'output_dir': 'output-7b-26k-lora', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 16, 'eval_accumulation_steps': 'None', 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 10, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'output-7b-26k-lora/runs/Feb08_18-45-12_default', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 100, 'save_total_limit': 100, 'save_safetensors': False, 'save_on_each_node': False, 'no_cuda': False, 'use_mps_device': False, 'seed': 42, 'data_seed': 'None', 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': 0, 'ddp_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 'None', 'dataloader_num_workers': 4, 'past_index': -1, 'run_name': 'output-7b-26k-lora', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'fsdp': '[]', 'fsdp_min_num_params': 0, 'fsdp_config': "{'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}", 'fsdp_transformer_layer_cls_to_wrap': 'None', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': 'None', 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': "['tensorboard', 'wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'ddp_broadcast_buffers': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'zzz99/output-7b-26k-lora', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'gradient_checkpointing': True, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': 'None', 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': 'None', 'torch_compile_mode': 'None', 'xpu_backend': 'None', 'cache_dir': 'None', 'model_max_length': 960, 'gradient_checkpointing_kwargs': "{'use_reentrant': False}", 'train_batch_size': 4, 'eval_batch_size': 1}
+2024-02-08 18:45:42,280 WARNING MsgRouterThr:608 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240208_185311-5uym8l7w/files/conda-environment.yaml

@@ -0,0 +1,662 @@
+name: base
+channels:
+  - file:///tmp/local_conda_forge_artifacts
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - _py-xgboost-mutex=2.0=gpu_0
+  - absl-py=2.0.0=pyhd8ed1ab_0
+  - accelerate=0.21.0=pyhd8ed1ab_0
+  - aiobotocore=2.7.0=pyhd8ed1ab_1
+  - aiohttp=3.9.1=py310h2372a71_0
+  - aioitertools=0.11.0=pyhd8ed1ab_0
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - aiosqlite=0.19.0=pyhd8ed1ab_0
+  - alsa-lib=1.2.10=hd590300_0
+  - altair=5.2.0=pyhd8ed1ab_0
+  - amazon-codewhisperer-jupyterlab-ext=2.0.1=pyhd8ed1ab_0
+  - amazon-sagemaker-jupyter-scheduler=3.0.5=pyhd8ed1ab_0
+  - ansi2html=1.8.0=py310hff52083_2
+  - ansiwrap=0.8.4=py_0
+  - antlr-python-runtime=4.9.3=pyhd8ed1ab_1
+  - anyio=3.7.1=pyhd8ed1ab_0
+  - aom=3.5.0=h27087fc_0
+  - appdirs=1.4.4=pyh9f0ad1d_0
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - astroid=3.0.1=py310hff52083_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - astunparse=1.6.3=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - async-timeout=4.0.3=pyhd8ed1ab_0
+  - atk-1.0=2.38.0=hd4edc92_1
+  - attr=2.5.1=h166bdaf_1
+  - attrs=23.1.0=pyh71513ae_1
+  - autogluon=0.8.2=pyhd8ed1ab_4
+  - autogluon.common=0.8.2=pyhd8ed1ab_4
+  - autogluon.core=0.8.2=light_py310h2d11d36_6
+  - autogluon.features=0.8.2=pyhd8ed1ab_3
+  - autogluon.multimodal=0.8.2=pyha770c72_5
+  - autogluon.tabular=0.8.2=pyha770c72_4
+  - autogluon.timeseries=0.8.2=pyhd8ed1ab_4
+  - autopep8=2.0.4=pyhd8ed1ab_0
+  - autovizwidget=0.21.0=pyh1a96a4e_0
+  - aws-c-auth=0.7.3=h28f7589_1
+  - aws-c-cal=0.6.1=hc309b26_1
+  - aws-c-common=0.9.0=hd590300_0
+  - aws-c-compression=0.2.17=h4d4d85c_2
+  - aws-c-event-stream=0.3.1=h2e3709c_4
+  - aws-c-http=0.7.11=h00aa349_4
+  - aws-c-io=0.13.32=he9a53bd_1
+  - aws-c-mqtt=0.9.3=hb447be9_1
+  - aws-c-s3=0.3.14=hf3aad02_1
+  - aws-c-sdkutils=0.1.12=h4d4d85c_1
+  - aws-checksums=0.1.17=h4d4d85c_1
+  - aws-crt-cpp=0.21.0=hb942446_5
+  - aws-embedded-metrics=3.2.0=pyhd8ed1ab_0
+  - aws-glue-sessions=1.0.4=pyhd8ed1ab_0
+  - aws-sdk-cpp=1.10.57=h85b1a90_19
+  - babel=2.13.1=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.2=pyha770c72_0
+  - binaryornot=0.4.4=py_1
+  - black=23.11.0=py310hff52083_0
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - blinker=1.7.0=pyhd8ed1ab_0
+  - blosc=1.21.5=h0f2a231_0
+  - boltons=23.0.0=pyhd8ed1ab_0
+  - boto3=1.28.64=pyhd8ed1ab_0
+  - botocore=1.31.64=pyhd8ed1ab_0
+  - brotli=1.0.9=h166bdaf_9
+  - brotli-bin=1.0.9=h166bdaf_9
+  - brotli-python=1.0.9=py310hd8f1fbe_9
+  - brunsli=0.1=h9c3ff4c_0
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.23.0=hd590300_0
+  - c-blosc2=2.11.3=hb4ffafa_0
+  - ca-certificates=2023.11.17=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cachetools=5.3.2=pyhd8ed1ab_0
+  - cairo=1.18.0=h3faef2a_0
+  - catalogue=2.0.10=py310hff52083_0
+  - catboost=1.2.2=py310hff52083_0
+  - certifi=2023.11.17=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - chardet=5.2.0=py310hff52083_1
+  - charls=2.4.2=h59595ed_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - click=8.1.7=unix_pyh707e725_0
+  - cloudpathlib=0.16.0=pyhd8ed1ab_0
+  - cloudpickle=2.2.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - conda=23.11.0=py310hff52083_1
+  - conda-libmamba-solver=23.11.1=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - confection=0.1.4=py310h17c5347_0
+  - contextlib2=21.6.0=pyhd8ed1ab_0
+  - contourpy=1.2.0=py310hd41b1e2_0
+  - cookiecutter=2.5.0=pyhca7485f_0
+  - croniter=1.4.1=pyhd8ed1ab_0
+  - cryptography=41.0.7=py310hb8475ec_1
+  - cuda-version=11.8=h70ddcb2_2
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudnn=8.8.0.121=hcdd5f01_4
+  - cycler=0.12.1=pyhd8ed1ab_0
+  - cymem=2.0.8=py310hc6cd4ac_1
+  - cyrus-sasl=2.1.27=h54b06d7_7
+  - cython-blis=0.7.10=py310h1f7b6fc_2
+  - cytoolz=0.12.2=py310h2372a71_1
+  - dash=2.14.2=pyhd8ed1ab_0
+  - dask-core=2023.12.0=pyhd8ed1ab_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - dataclasses-json=0.6.3=pyhd8ed1ab_0
+  - datasets=2.14.7=pyhd8ed1ab_0
+  - dav1d=1.2.1=hd590300_0
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.8.0=py310hc6cd4ac_1
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - deepmerge=1.1.0=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - dill=0.3.7=pyhd8ed1ab_0
+  - distributed=2023.12.0=pyhd8ed1ab_0
+  - distro=1.8.0=pyhd8ed1ab_0
+  - docker-pycreds=0.4.0=py_0
+  - docstring-to-markdown=0.13=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - et_xmlfile=1.1.0=pyhd8ed1ab_0
+  - evaluate=0.4.1=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_0
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.5.0=hcb278e6_1
+  - faiss=1.7.4=py310h9ed8947_0_cpu
+  - faiss-cpu=1.7.4=h718b53a_0
+  - fastai=2.7.13=pyhd8ed1ab_0
+  - fastcore=1.5.29=pyhd8ed1ab_0
+  - fastdownload=0.0.7=pyhd8ed1ab_0
+  - fastprogress=1.0.3=pyhd8ed1ab_0
+  - filelock=3.13.1=pyhd8ed1ab_0
+  - flake8=6.1.0=pyhd8ed1ab_0
+  - flask=3.0.0=pyhd8ed1ab_0
+  - flatbuffers=23.5.26=h59595ed_1
+  - fmt=10.1.1=h00ab1b0_1
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=h77eed37_1
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.46.0=py310h2372a71_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h267a509_2
+  - fribidi=1.0.10=h36c2ea0_0
+  - frozenlist=1.4.0=py310h2372a71_1
+  - fsspec=2023.6.0=pyh1a96a4e_0
+  - future=0.18.3=pyhd8ed1ab_0
+  - gast=0.4.0=pyh9f0ad1d_0
+  - gdk-pixbuf=2.42.10=h6b639ba_2
+  - gdown=4.7.1=pyhd8ed1ab_0
+  - gettext=0.21.1=h27087fc_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h0b41bf4_3
+  - gitdb=4.0.11=pyhd8ed1ab_0
+  - gitpython=3.1.40=pyhd8ed1ab_0
+  - glib=2.78.3=hfc55251_0
+  - glib-tools=2.78.3=hfc55251_0
+  - glog=0.6.0=h6f12383_0
+  - gluonts=0.13.7=pyhd8ed1ab_0
+  - gmp=6.3.0=h59595ed_0
+  - gmpy2=2.1.2=py310h3ec546c_1
+  - google-api-core=2.15.0=pyhd8ed1ab_0
+  - google-auth=2.25.2=pyhca7485f_0
+  - google-auth-oauthlib=1.0.0=pyhd8ed1ab_1
+  - google-pasta=0.2.0=pyh8c360ce_0
+  - googleapis-common-protos=1.62.0=pyhd8ed1ab_0
+  - graphite2=1.3.13=h58526e2_1001
+  - graphviz=8.1.0=h28d9a01_0
+  - grpcio=1.54.3=py310heca2aa9_0
+  - gst-plugins-base=1.22.7=h8e1006c_1
+  - gstreamer=1.22.7=h98fc4e7_1
+  - gtk2=2.24.33=h90689f9_2
+  - gts=0.7.6=h977cf35_4
+  - h5py=3.10.0=nompi_py310ha2ad45a_100
+  - harfbuzz=8.3.0=h3d44ed6_0
+  - hdf5=1.14.2=nompi_h4f84152_100
+  - hdijupyterutils=0.21.0=pyh1a96a4e_0
+  - huggingface_hub=0.19.4=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - imagecodecs=2023.8.12=py310hc929067_0
+  - imageio=2.31.5=pyh8c1a49c_0
+  - importlib-metadata=6.10.0=pyha770c72_0
+  - importlib_metadata=6.10.0=hd8ed1ab_0
+  - importlib_resources=6.1.1=pyhd8ed1ab_0
+  - iniconfig=2.0.0=pyhd8ed1ab_0
+  - ipykernel=6.26.0=pyhf8b6a83_0
+  - ipython=8.18.1=pyh707e725_3
+  - ipywidgets=8.1.1=pyhd8ed1ab_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - isort=5.12.0=pyhd8ed1ab_1
+  - itsdangerous=2.1.2=pyhd8ed1ab_0
+  - jax=0.4.20=pyhd8ed1ab_0
+  - jaxlib=0.4.14=cpu_py310h67d73b5_1
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.3.2=pyhd8ed1ab_0
+  - json5=0.9.14=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpath-ng=1.6.0=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.17.3=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.17.3=pyhd8ed1ab_0
+  - jupyter=1.0.0=pyhd8ed1ab_10
+  - jupyter-ai=2.6.0=pyhd8ed1ab_2
+  - jupyter-ai-magics=2.6.0=pyhd8ed1ab_1
+  - jupyter-dash=0.4.2=pyhd8ed1ab_1
+  - jupyter-lsp=2.2.1=pyhd8ed1ab_0
+  - jupyter-server-mathjax=0.2.6=pyh5bfe37b_1
+  - jupyter-server-proxy=4.1.0=pyhd8ed1ab_0
+  - jupyter_client=8.6.0=pyhd8ed1ab_0
+  - jupyter_console=6.6.3=pyhd8ed1ab_0
+  - jupyter_core=5.5.0=py310hff52083_0
+  - jupyter_events=0.6.3=pyhd8ed1ab_1
+  - jupyter_scheduler=2.4.0=pyha770c72_1
+  - jupyter_server=2.10.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.0=pyhd8ed1ab_0
+  - jupyterlab=4.0.9=pyhd8ed1ab_0
+  - jupyterlab-git=0.50.0=pyhd8ed1ab_0
+  - jupyterlab-lsp=5.0.1=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_0
+  - jupyterlab_server=2.24.0=pyhd8ed1ab_0
+  - jupyterlab_widgets=3.0.9=pyhd8ed1ab_0
+  - jxrlib=1.1=h7f98852_2
+  - keras=2.12.0=pyhd8ed1ab_0
+  - keras-preprocessing=1.1.2=pyhd8ed1ab_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.5=py310hd41b1e2_1
+  - lame=3.100=h166bdaf_1003
+  - langchain=0.0.318=pyhd8ed1ab_0
+  - langcodes=3.3.0=pyhd8ed1ab_0
+  - langsmith=0.0.69=pyhd8ed1ab_0
+  - lcms2=2.15=haa2dc70_1
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20230125.3=cxx17_h59595ed_0
+  - libaec=1.1.2=h59595ed_1
+  - libarchive=3.7.2=h039dbb9_0
+  - libarrow=12.0.1=hb87d912_8_cpu
+  - libavif=0.11.1=h8182462_2
+  - libblas=3.9.0=16_linux64_mkl
+  - libbrotlicommon=1.0.9=h166bdaf_9
+  - libbrotlidec=1.0.9=h166bdaf_9
+  - libbrotlienc=1.0.9=h166bdaf_9
+  - libcap=2.69=h0f662aa_0
+  - libcblas=3.9.0=16_linux64_mkl
+  - libclang=15.0.7=default_hb11cfb5_4
+  - libclang13=15.0.7=default_ha2b6cf4_4
+  - libcrc32c=1.1.2=h9c3ff4c_0
+  - libcups=2.3.3=h4637d8d_4
+  - libcurl=8.4.0=hca28451_0
+  - libdeflate=1.18=h0b41bf4_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libevent=2.1.12=hf998b51_1
+  - libexpat=2.5.0=hcb278e6_1
+  - libfaiss=1.7.4=hf47d654_0_cpu
+  - libfaiss-avx2=1.7.4=h1234567_0_cpu
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.3=h59595ed_0
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgcrypt=1.10.3=hd590300_0
+  - libgd=2.3.3=h74d50f4_7
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libglib=2.78.3=h783c2da_0
+  - libgoogle-cloud=2.12.0=hac9eb74_1
+  - libgpg-error=1.47=h71f35ed_0
+  - libgrpc=1.54.3=hb20ce57_0
+  - libhwloc=2.9.3=default_h554bfaf_1009
+  - libiconv=1.17=hd590300_1
+  - libjpeg-turbo=2.1.5.1=hd590300_1
+  - liblapack=3.9.0=16_linux64_mkl
+  - libllvm14=14.0.6=hcd5def8_4
+  - libllvm15=15.0.7=h5cf9203_3
+  - libmagma=2.7.1=hc72dce7_6
+  - libmagma_sparse=2.7.1=h8354cda_6
+  - libmamba=1.5.3=had39da4_2
+  - libmambapy=1.5.3=py310h39ff949_2
+  - libnghttp2=1.58.0=h47da74e_0
+  - libnsl=2.0.1=hd590300_0
+  - libntlm=1.4=h7f98852_1002
+  - libnuma=2.0.16=h0b41bf4_1
+  - libogg=1.3.4=h7f98852_1
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.39=h753d276_0
+  - libpq=15.4=hfc447b1_2
+  - libprotobuf=3.21.12=hfc55251_2
+  - librsvg=2.56.3=h98fae49_0
+  - libsecret=0.18.8=h329b89f_2
+  - libsentencepiece=0.1.99=h180e1df_0
+  - libsndfile=1.2.2=hc60ed4a_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libsystemd0=255=h3516f8a_0
+  - libthrift=0.18.1=h8fd135c_2
+  - libtiff=4.5.1=h8b53f26_1
+  - libtool=2.4.7=h27087fc_0
+  - libutf8proc=2.8.0=h166bdaf_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libuv=1.46.0=hd590300_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libwebp=1.3.1=hbf2b3c1_0
+  - libwebp-base=1.3.1=hd590300_0
+  - libxcb=1.15=h0b41bf4_0
+  - libxgboost=1.7.6=cuda118_hd3b444d_6
+  - libxkbcommon=1.6.0=h5d7e998_0
+  - libxml2=2.11.6=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - libzopfli=1.0.3=h9c3ff4c_0
+  - lightgbm=3.3.5=py310heca2aa9_0
+  - lightning-utilities=0.10.0=pyhd8ed1ab_0
+  - llvm-openmp=17.0.6=h4dfa4b3_0
+  - llvmlite=0.41.1=py310h1b8f574_0
+  - locket=1.0.0=pyhd8ed1ab_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - magma=2.7.1=ha770c72_6
+  - markdown=3.5.1=pyhd8ed1ab_0
+  - markdown-it-py=3.0.0=pyhd8ed1ab_0
+  - markupsafe=2.1.3=py310h2372a71_1
+  - marshmallow=3.20.1=pyhd8ed1ab_0
+  - matplotlib=3.8.2=py310hff52083_0
+  - matplotlib-base=3.8.2=py310h62c0568_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mccabe=0.7.0=pyhd8ed1ab_0
+  - mdurl=0.1.0=pyhd8ed1ab_0
+  - menuinst=2.0.1=py310hff52083_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2022.2.1=h84fe81f_16997
+  - ml_dtypes=0.3.1=py310hcc13569_2
+  - mlforecast=0.7.3=pyhd8ed1ab_0
+  - mock=5.1.0=pyhd8ed1ab_0
+  - model-index=0.1.11=pyhd8ed1ab_0
+  - mpc=1.3.1=hfe3b2da_0
+  - mpfr=4.2.1=h9458935_0
+  - mpg123=1.32.3=h59595ed_0
+  - mpmath=1.3.0=pyhd8ed1ab_0
+  - msgpack-python=1.0.7=py310hd41b1e2_0
+  - multidict=6.0.4=py310h2372a71_1
+  - multiprocess=0.70.15=py310h2372a71_1
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.10=py310hc6cd4ac_1
+  - mypy_extensions=1.0.0=pyha770c72_0
+  - mysql-common=8.0.33=hf1915f5_6
+  - mysql-libs=8.0.33=hca2cd23_6
+  - nbclient=0.8.0=pyhd8ed1ab_0
+  - nbconvert=7.12.0=pyhd8ed1ab_0
+  - nbconvert-core=7.12.0=pyhd8ed1ab_0
+  - nbconvert-pandoc=7.12.0=pyhd8ed1ab_0
+  - nbdime=4.0.1=pyhd8ed1ab_0
+  - nbformat=5.9.2=pyhd8ed1ab_0
+  - nccl=2.19.4.1=h6103f9b_0
+  - ncurses=6.4=h59595ed_2
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - networkx=3.2.1=pyhd8ed1ab_0
+  - nlpaug=1.1.11=pyhd8ed1ab_1
+  - nltk=3.8.1=pyhd8ed1ab_0
+  - nodejs=18.18.2=hb753e55_1
+  - nose=1.3.7=py_1006
+  - notebook=7.0.6=pyhd8ed1ab_0
+  - notebook-shim=0.2.3=pyhd8ed1ab_0
+  - nptyping=2.4.1=pyhd8ed1ab_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.95=h1d7d5a4_0
+  - numba=0.58.1=py310h7dc5dd1_0
+  - numexpr=2.8.7=mkl_py310h5e07f1c_4
+  - numpy=1.26.2=py310hb13e2d6_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - omegaconf=2.2.3=pyhd8ed1ab_0
+  - openai=0.28.1=pyhd8ed1ab_0
+  - openapi-schema-pydantic=1.2.4=pyhd8ed1ab_0
+  - openjpeg=2.5.0=hfec8fc6_2
+  - openmim=0.3.7=pyhd8ed1ab_0
+  - openpyxl=3.1.2=py310h2372a71_1
+  - openssl=3.1.4=hd590300_0
+  - opt-einsum=3.3.0=hd8ed1ab_2
+  - opt_einsum=3.3.0=pyhc1e730c_2
+  - orc=1.9.0=h2f23424_1
+  - ordered-set=4.1.0=pyhd8ed1ab_0
+  - overrides=7.4.0=pyhd8ed1ab_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pandas=2.1.4=py310hcc13569_0
+  - pandas-stubs=2.1.1.230928=pyhd8ed1ab_1
+  - pandoc=3.1.3=h32600fe_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - pango=1.50.14=ha41ecd1_2
+  - papermill=2.4.0=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - partd=1.4.1=pyhd8ed1ab_0
+  - pathos=0.3.1=pyhd8ed1ab_0
+  - pathspec=0.12.1=pyhd8ed1ab_0
+  - pathtools=0.1.2=py_1
+  - pathy=0.10.2=pyhd8ed1ab_0
+  - patsy=0.5.4=pyhd8ed1ab_0
+  - pcre2=10.42=hcad00b1_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=9.5.0=py310h582fbeb_1
+  - pip=23.3.1=pyhd8ed1ab_0
+  - pixman=0.42.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - plotly=5.18.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - ply=3.11=py_1
+  - pox=0.3.3=pyhd8ed1ab_0
+  - ppft=1.7.6.7=pyhd8ed1ab_0
+  - preshed=3.0.9=py310hc6cd4ac_1
+  - prometheus_client=0.19.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.41=pyha770c72_0
+  - prompt_toolkit=3.0.41=hd8ed1ab_0
+  - protobuf=4.21.12=py310heca2aa9_0
+  - psutil=5.9.5=py310h2372a71_1
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio-client=16.1=hb77b528_5
+  - pure-sasl=0.6.2=pyhd8ed1ab_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - py-xgboost=1.7.6=cuda118_py310h9840055_6
+  - py-xgboost-gpu=1.7.6=py310h9840055_6
+  - pyarrow=12.0.1=py310h0576679_8_cpu
+  - pyarrow-hotfix=0.6=pyhd8ed1ab_0
+  - pyasn1=0.5.1=pyhd8ed1ab_0
+  - pyasn1-modules=0.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycodestyle=2.11.1=pyhd8ed1ab_0
+  - pycosat=0.6.6=py310h2372a71_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydocstyle=6.3.0=pyhd8ed1ab_0
+  - pyflakes=3.1.0=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pyhive=0.7.0=pyhd8ed1ab_0
+  - pyjwt=2.8.0=pyhd8ed1ab_0
+  - pylint=3.0.3=pyhd8ed1ab_0
+  - pyopenssl=23.3.0=pyhd8ed1ab_0
+  - pyparsing=3.1.1=pyhd8ed1ab_0
+  - pyqt=5.15.9=py310h04931ad_5
+  - pyqt5-sip=12.12.2=py310hc6cd4ac_5
+  - pyrsistent=0.20.0=py310h2372a71_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - pyspnego=0.9.1=py310h2372a71_2
+  - pytesseract=0.3.10=pyhd8ed1ab_0
+  - pytest=7.4.3=pyhd8ed1ab_0
+  - pytest-subtests=0.11.0=pyhd8ed1ab_0
+  - python=3.10.13=hd12c33a_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.0=pyhd8ed1ab_0
+  - python-flatbuffers=23.5.26=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=pyh22cad53_0
+  - python-gssapi=1.8.3=py310h98feede_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python-lsp-jsonrpc=1.1.2=pyhd8ed1ab_0
+  - python-lsp-server=1.9.0=pyhd8ed1ab_0
+  - python-lsp-server-base=1.9.0=pyhd8ed1ab_0
+  - python-slugify=8.0.1=pyhd8ed1ab_2
+  - python-tzdata=2023.3=pyhd8ed1ab_0
+  - python-xxhash=3.4.1=py310h2372a71_0
+  - python_abi=3.10=4_cp310
+  - pytoolconfig=1.2.5=pyhd8ed1ab_0
+  - pytorch=2.0.0=cuda112py310he33e0d6_200
+  - pytorch-gpu=2.0.0=cuda112py310h9871d0b_200
+  - pytorch-lightning=2.0.9=pyhd8ed1ab_0
+  - pytorch-metric-learning=1.7.3=pyhd8ed1ab_0
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywavelets=1.4.1=py310h1f7b6fc_1
+  - pyyaml=6.0.1=py310h2372a71_1
+  - pyzmq=25.1.2=py310h795f18f_0
+  - qt-main=5.15.8=hc47bfe8_16
+  - qtconsole-base=5.5.1=pyha770c72_0
+  - qtpy=2.4.1=pyhd8ed1ab_0
+  - rdma-core=28.9=h59595ed_1
+  - re2=2023.03.02=h8c504da_0
+  - readline=8.2=h8228510_1
+  - regex=2023.10.3=py310h2372a71_0
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - requests-kerberos=0.14.0=pyhd8ed1ab_1
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - responses=0.18.0=pyhd8ed1ab_0
+  - retrying=1.3.3=py_2
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rich=13.7.0=pyhd8ed1ab_0
+  - ripgrep=14.0.3=he8a937b_1
+  - rope=1.11.0=pyhd8ed1ab_1
+  - rsa=4.9=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py310h2372a71_0
+  - ruamel.yaml.clib=0.2.7=py310h2372a71_2
+  - s2n=1.3.49=h06160fa_0
+  - s3transfer=0.7.0=pyhd8ed1ab_1
+  - sacremoses=0.0.53=pyhd8ed1ab_0
+  - safetensors=0.3.3=py310hcb5633a_1
+  - sagemaker-code-editor=1.83.1=ha1e3dc1_3
+  - sagemaker-headless-execution-driver=0.0.12=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-emr-extension=0.1.9=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension=0.2.0=pyhd8ed1ab_0
+  - sagemaker-jupyterlab-extension-common=0.1.9=pyhd8ed1ab_0
+  - sagemaker-kernel-wrapper=0.0.2=pyhd8ed1ab_0
+  - sagemaker-python-sdk=2.198.0=pyhd8ed1ab_0
+  - sagemaker-studio-analytics-extension=0.0.21=pyhd8ed1ab_0
+  - sagemaker-studio-sparkmagic-lib=0.1.4=pyhd8ed1ab_0
+  - sasl=0.3.1=py310hd8f1fbe_2
+  - schema=0.7.5=pyhd8ed1ab_1
+  - scikit-image=0.19.3=py310h769672d_2
+  - scikit-learn=1.3.2=py310h1fdf081_2
+  - scipy=1.11.4=py310hb13e2d6_0
+  - send2trash=1.8.2=pyh41d4057_0
+  - sentencepiece=0.1.99=hff52083_0
+  - sentencepiece-python=0.1.99=py310hebdb9f0_0
+  - sentencepiece-spm=0.1.99=h180e1df_0
+  - sentry-sdk=1.38.0=pyhd8ed1ab_0
+  - seqeval=1.2.2=pyhd3deb0d_0
+  - setproctitle=1.3.3=py310h2372a71_0
+  - setuptools=68.2.2=pyhd8ed1ab_0
+  - shellingham=1.5.4=pyhd8ed1ab_0
+  - simpervisor=1.0.0=pyhd8ed1ab_0
+  - sip=6.7.12=py310hc6cd4ac_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sleef=3.5.1=h9b69904_2
+  - smart_open=5.2.1=pyhd8ed1ab_0
+  - smdebug-rulesconfig=1.0.1=pyhd3deb0d_1
+  - smmap=5.0.0=pyhd8ed1ab_0
+  - snappy=1.1.10=h9fff704_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - snowballstemmer=2.2.0=pyhd8ed1ab_0
+  - sortedcontainers=2.4.0=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - spacy=3.7.2=py310hcb52e73_0
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.5=pyhd8ed1ab_0
+  - sparkmagic=0.21.0=pyhd8ed1ab_0
+  - srsly=2.4.8=py310hc6cd4ac_1
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - statsforecast=1.4.0=pyhd8ed1ab_0
+  - statsmodels=0.14.0=py310h1f7b6fc_3
+  - supervisor=4.2.5=py310hff52083_2
+  - sympy=1.12=pypyh9d50eac_103
+  - tabulate=0.9.0=pyhd8ed1ab_1
+  - tbb=2021.11.0=h00ab1b0_0
+  - tblib=1.7.0=pyhd8ed1ab_0
+  - tenacity=8.2.3=pyhd8ed1ab_0
+  - tensorboard=2.12.3=pyhd8ed1ab_0
+  - tensorboard-data-server=0.7.0=py310h75e40e8_1
+  - tensorflow=2.12.1=cuda112py310h457873b_1
+  - tensorflow-base=2.12.1=cuda112py310h1d65785_1
+  - tensorflow-estimator=2.12.1=cuda112py310ha5e6de5_1
+  - termcolor=2.3.0=pyhd8ed1ab_0
+  - terminado=0.18.0=pyh0d859eb_0
+  - text-unidecode=1.3=pyhd8ed1ab_1
+  - textwrap3=0.9.2=py_0
+  - thinc=8.2.1=py310hcb52e73_0
+  - threadpoolctl=3.2.0=pyha21a80b_0
+  - thrift=0.19.0=py310hc6cd4ac_1
+  - thrift_sasl=0.4.3=pyhd8ed1ab_2
+  - tifffile=2023.12.9=pyhd8ed1ab_0
+  - tiktoken=0.5.2=py310h47bb294_0
+  - timm=0.9.12=pyhd8ed1ab_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tokenizers=0.13.3=py310he1f1126_0
+  - toml=0.10.2=pyhd8ed1ab_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tomlkit=0.12.3=pyha770c72_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchmetrics=1.0.3=pyhd8ed1ab_0
+  - torchvision=0.15.2=cuda112py310h0801bf5_1
+  - tornado=6.3.3=py310h2372a71_1
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - traitlets=5.14.0=pyhd8ed1ab_0
+  - transformers=4.31.0=pyhd8ed1ab_0
+  - truststore=0.8.0=pyhd8ed1ab_0
+  - typer=0.9.0=pyhd8ed1ab_0
+  - types-python-dateutil=2.8.19.14=pyhd8ed1ab_0
+  - types-pytz=2023.3.1.1=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - typing_inspect=0.9.0=pyhd8ed1ab_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - typish=1.9.3=pyhd8ed1ab_0
+  - tzdata=2023c=h71feb2d_0
+  - ucx=1.14.1=h64cca9d_5
+  - ujson=5.9.0=py310hc6cd4ac_0
+  - unicodedata2=15.1.0=py310h2372a71_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - urllib3=1.26.18=pyhd8ed1ab_0
+  - wandb=0.16.1=pyhd8ed1ab_0
+  - wasabi=1.1.2=py310hff52083_0
+  - wcwidth=0.2.12=pyhd8ed1ab_0
+  - weasel=0.3.4=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.7.0=pyhd8ed1ab_0
+  - werkzeug=3.0.1=pyhd8ed1ab_0
+  - whatthepatch=1.0.5=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - widgetsnbextension=4.0.9=pyhd8ed1ab_0
+  - window-ops=0.0.14=pyhd8ed1ab_0
+  - wrapt=1.16.0=py310h2372a71_0
+  - xcb-util=0.4.0=hd590300_1
+  - xcb-util-image=0.4.0=h8ee46fc_1
+  - xcb-util-keysyms=0.4.0=h8ee46fc_1
+  - xcb-util-renderutil=0.3.9=hd590300_1
+  - xcb-util-wm=0.4.1=h8ee46fc_1
+  - xgboost=1.7.6=cuda118_py310h9840055_6
+  - xkeyboard-config=2.40=hd590300_0
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.7=h8ee46fc_0
+  - xorg-libxau=1.0.11=hd590300_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xf86vidmodeproto=2.3.1=h7f98852_1002
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xxhash=0.8.2=hd590300_0
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yaml-cpp=0.8.0=h59595ed_0
+  - yapf=0.40.1=pyhd8ed1ab_0
+  - yarl=1.9.3=py310h2372a71_0
+  - zeromq=4.3.5=h59595ed_0
+  - zfp=1.0.0=h59595ed_4
+  - zict=3.0.0=pyhd8ed1ab_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zlib-ng=2.0.7=h0b41bf4_0
+  - zstandard=0.22.0=py310h1275a96_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - annotated-types==0.6.0
+      - greenlet==3.0.1
+      - jupyter-scheduler==2.4.0
+      - krb5==0.5.1
+      - peft==0.8.2
+      - pydantic==2.5.1
+      - pydantic-core==2.14.3
+      - sqlalchemy==1.4.50
+prefix: /opt/conda

wandb/run-20240208_185311-5uym8l7w/files/config.yaml

@@ -0,0 +1,614 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.13
+    cli_version: 0.16.1
+    framework: huggingface
+    huggingface_version: 4.31.0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1707418391.295585
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 15
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 98
+      3:
+      - 7
+      - 23
+      4: 3.10.13
+      5: 0.16.1
+      6: 4.31.0
+      8:
+      - 5
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+vocab_size:
+  desc: null
+  value: 102400
+max_position_embeddings:
+  desc: null
+  value: 4096
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 11008
+num_hidden_layers:
+  desc: null
+  value: 30
+num_attention_heads:
+  desc: null
+  value: 32
+num_key_value_heads:
+  desc: null
+  value: 32
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+pretraining_tp:
+  desc: null
+  value: 1
+use_cache:
+  desc: null
+  value: true
+rope_scaling:
+  desc: null
+  value: null
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 100000
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 100015
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: deepseek-ai/deepseek-coder-7b-instruct-v1.5
+transformers_version:
+  desc: null
+  value: 4.31.0
+attention_bias:
+  desc: null
+  value: false
+model_type:
+  desc: null
+  value: llama
+rope_theta:
+  desc: null
+  value: 10000.0
+output_dir:
+  desc: null
+  value: output-7b-26k-lora
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_device_eval_batch_size:
+  desc: null
+  value: 1
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 2.0e-05
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 2
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 10
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: output-7b-26k-lora/runs/Feb08_18-53-01_default
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 1
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_steps:
+  desc: null
+  value: 100
+save_total_limit:
+  desc: null
+  value: 100
+save_safetensors:
+  desc: null
+  value: false
+save_on_each_node:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: None
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: None
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: None
+tpu_num_cores:
+  desc: null
+  value: None
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: '[]'
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: None
+dataloader_num_workers:
+  desc: null
+  value: 4
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: output-7b-26k-lora
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: None
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: None
+greater_is_better:
+  desc: null
+  value: None
+ignore_data_skip:
+  desc: null
+  value: false
+sharded_ddp:
+  desc: null
+  value: '[]'
+fsdp:
+  desc: null
+  value: '[]'
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value: '{''fsdp_min_num_params'': 0, ''xla'': False, ''xla_fsdp_grad_ckpt'': False}'
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: None
+deepspeed:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: None
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value: '[''tensorboard'', ''wandb'']'
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+ddp_broadcast_buffers:
+  desc: null
+  value: None
+dataloader_pin_memory:
+  desc: null
+  value: true
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: None
+hub_model_id:
+  desc: null
+  value: zzz99/output-7b-26k-lora
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: None
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: None
+torch_compile_mode:
+  desc: null
+  value: None
+xpu_backend:
+  desc: null
+  value: None
+cache_dir:
+  desc: null
+  value: None
+model_max_length:
+  desc: null
+  value: 960
+gradient_checkpointing_kwargs:
+  desc: null
+  value: '{''use_reentrant'': False}'
+train_batch_size:
+  desc: null
+  value: 4
+eval_batch_size:
+  desc: null
+  value: 1

wandb/run-20240208_185311-5uym8l7w/files/output.log

@@ -0,0 +1,21 @@
+  0%|                                                                                         | 0/2 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None
+  warnings.warn("None of the inputs have requires_grad=True. Gradients will be None")
+Traceback (most recent call last):
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py", line 236, in <module>
+    train()
+  File "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py", line 223, in train
+    trainer.train()
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+    return inner_training_loop(
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
+    tr_loss_step = self.training_step(model, inputs)
+  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2665, in training_step
+    self.accelerator.backward(loss)
+  File "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py", line 1853, in backward
+    loss.backward(**kwargs)
+  File "/opt/conda/lib/python3.10/site-packages/torch/_tensor.py", line 487, in backward
+    torch.autograd.backward(
+  File "/opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py", line 200, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20240208_185311-5uym8l7w/files/requirements.txt

@@ -0,0 +1,419 @@
+absl-py==2.0.0
+accelerate==0.21.0
+aiobotocore==2.7.0
+aiohttp==3.9.1
+aioitertools==0.11.0
+aiosignal==1.3.1
+aiosqlite==0.19.0
+altair==5.2.0
+amazon-codewhisperer-jupyterlab-ext==2.0.1
+amazon-sagemaker-jupyter-scheduler==3.0.5
+annotated-types==0.6.0
+ansi2html==0.0.0
+ansiwrap==0.8.4
+antlr4-python3-runtime==4.9.3
+anyio==3.7.1
+appdirs==1.4.4
+archspec==0.2.2
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+astroid==3.0.1
+asttokens==2.4.1
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+autogluon.common==0.8.2
+autogluon.core==0.8.2
+autogluon.features==0.8.2
+autogluon.multimodal==0.8.2
+autogluon.tabular==0.8.2
+autogluon.timeseries==0.8.2
+autogluon==0.8.2
+autopep8==2.0.4
+autovizwidget==0.21.0
+aws-embedded-metrics==3.2.0
+aws-glue-sessions==1.0.4
+babel==2.13.1
+beautifulsoup4==4.12.2
+binaryornot==0.4.4
+black==23.11.0
+bleach==6.1.0
+blinker==1.7.0
+blis==0.7.10
+boltons==23.0.0
+boto3==1.28.64
+botocore==1.31.64
+brotli==1.0.9
+cached-property==1.5.2
+cachetools==5.3.2
+catalogue==2.0.10
+catboost==1.2.2
+certifi==2023.11.17
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==2.2.1
+colorama==0.4.6
+comm==0.1.4
+conda-libmamba-solver==23.11.1
+conda-package-handling==2.2.0
+conda-package-streaming==0.9.0
+conda==23.11.0
+confection==0.1.4
+contextlib2==21.6.0
+contourpy==1.2.0
+cookiecutter==2.5.0
+croniter==1.4.1
+cryptography==41.0.7
+cycler==0.12.1
+cymem==2.0.8
+cytoolz==0.12.2
+dash==2.14.2
+dask==2023.12.0
+dataclasses-json==0.6.3
+dataclasses==0.8
+datasets==2.14.7
+debugpy==1.8.0
+decorator==5.1.1
+deepmerge==1.1.0
+defusedxml==0.7.1
+dill==0.3.7
+distributed==2023.12.0
+distro==1.8.0
+docker-pycreds==0.4.0
+docstring-to-markdown==0.13
+entrypoints==0.4
+et-xmlfile==1.1.0
+evaluate==0.4.1
+exceptiongroup==1.2.0
+executing==2.0.1
+faiss==1.7.4
+fastai==2.7.13
+fastcore==1.5.29
+fastdownload==0.0.7
+fastjsonschema==2.19.0
+fastprogress==1.0.3
+filelock==3.13.1
+flake8==6.1.0
+flask==3.0.0
+flatbuffers==23.5.26
+fonttools==4.46.0
+fqdn==1.5.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+future==0.18.3
+gast==0.4.0
+gdown==4.7.1
+gitdb==4.0.11
+gitpython==3.1.40
+gluonts==0.13.7
+gmpy2==2.1.2
+google-api-core==2.15.0
+google-auth-oauthlib==1.0.0
+google-auth==2.25.2
+google-pasta==0.2.0
+googleapis-common-protos==1.62.0
+graphviz==0.20.1
+greenlet==3.0.1
+grpcio==1.54.3
+gssapi==1.8.3
+h5py==3.10.0
+hdijupyterutils==0.21.0
+huggingface-hub==0.19.4
+idna==3.6
+imagecodecs==2023.8.12
+imageio==2.31.5
+importlib-metadata==6.10.0
+importlib-resources==6.1.1
+iniconfig==2.0.0
+ipykernel==6.26.0
+ipython==8.18.1
+ipywidgets==8.1.1
+isoduration==20.11.0
+isort==5.12.0
+itsdangerous==2.1.2
+jax==0.4.20
+jaxlib==0.4.14
+jedi==0.19.1
+jinja2==3.1.2
+jmespath==1.0.1
+joblib==1.3.2
+json5==0.9.14
+jsonpatch==1.33
+jsonpath-ng==1.6.0
+jsonpointer==2.4
+jsonschema==4.17.3
+jupyter-ai-magics==2.6.0
+jupyter-ai==2.6.0
+jupyter-client==8.6.0
+jupyter-console==6.6.3
+jupyter-core==5.5.0
+jupyter-dash==0.4.2
+jupyter-events==0.6.3
+jupyter-lsp==2.2.1
+jupyter-scheduler==2.4.0
+jupyter-server-mathjax==0.2.6
+jupyter-server-proxy==4.1.0
+jupyter-server-terminals==0.5.0
+jupyter-server==2.10.0
+jupyter==1.0.0
+jupyterlab-git==0.50.0
+jupyterlab-lsp==5.0.1
+jupyterlab-pygments==0.3.0
+jupyterlab-server==2.24.0
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+keras-preprocessing==1.1.2
+keras==2.12.0
+kiwisolver==1.4.5
+krb5==0.5.1
+langchain==0.0.318
+langcodes==3.3.0
+langsmith==0.0.69
+libmambapy==1.5.3
+lightgbm==3.3.5
+lightning-utilities==0.10.0
+llvmlite==0.41.1
+locket==1.0.0
+markdown-it-py==3.0.0
+markdown==3.5.1
+markupsafe==2.1.3
+marshmallow==3.20.1
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mccabe==0.7.0
+mdurl==0.1.0
+menuinst==2.0.1
+mistune==3.0.2
+ml-dtypes==0.3.1
+mlforecast==0.7.3
+mock==5.1.0
+model-index==0.1.11
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+munkres==1.1.4
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.12.0
+nbdime==4.0.1
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.2.1
+nlpaug==1.1.11
+nltk==3.8.1
+nose==1.3.7
+notebook-shim==0.2.3
+notebook==7.0.6
+nptyping==2.4.1
+numba==0.58.1
+numexpr==2.8.7
+numpy==1.26.2
+oauthlib==3.2.2
+omegaconf==2.2.3
+openai==0.28.1
+openapi-schema-pydantic==1.2.4
+openmim==0.3.7
+openpyxl==3.1.2
+opt-einsum==3.3.0
+ordered-set==4.1.0
+overrides==7.4.0
+packaging==23.2
+pandas-stubs==2.1.1.230928
+pandas==2.1.4
+pandocfilters==1.5.0
+papermill==2.4.0
+parso==0.8.3
+partd==1.4.1
+pathos==0.3.1
+pathspec==0.12.1
+pathtools==0.1.2
+pathy==0.10.2
+patsy==0.5.4
+peft==0.8.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.5.0
+pip==23.3.1
+pkgutil-resolve-name==1.3.10
+platformdirs==4.1.0
+plotly==5.18.0
+pluggy==1.3.0
+ply==3.11
+pox==0.3.3
+ppft==1.7.6.7
+preshed==3.0.9
+prometheus-client==0.19.0
+prompt-toolkit==3.0.41
+protobuf==4.21.12
+psutil==5.9.5
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pure-sasl==0.6.2
+pyarrow-hotfix==0.6
+pyarrow==12.0.1
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pycodestyle==2.11.1
+pycosat==0.6.6
+pycparser==2.21
+pydantic-core==2.14.3
+pydantic==1.10.13
+pydocstyle==6.3.0
+pyflakes==3.1.0
+pygments==2.17.2
+pyhive==0.7.0
+pyjwt==2.8.0
+pylint==3.0.3
+pyopenssl==23.3.0
+pyparsing==3.1.1
+pyqt5-sip==12.12.2
+pyqt5==5.15.9
+pyrsistent==0.20.0
+pysocks==1.7.1
+pyspnego==0.9.1
+pytesseract==0.3.10
+pytest-subtests==0.11.0
+pytest==7.4.3
+python-dateutil==2.8.2
+python-json-logger==2.0.7
+python-lsp-jsonrpc==1.1.2
+python-lsp-server==1.9.0
+python-slugify==8.0.1
+pytoolconfig==1.2.5
+pytorch-lightning==2.0.9
+pytorch-metric-learning==1.7.3
+pytz==2023.3
+pyu2f==0.1.5
+pywavelets==1.4.1
+pyyaml==6.0.1
+pyzmq==25.1.2
+qtconsole==5.5.1
+qtpy==2.4.1
+regex==2023.10.3
+requests-kerberos==0.14.0
+requests-oauthlib==1.3.1
+requests==2.31.0
+responses==0.18.0
+retrying==1.3.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.0
+rope==1.11.0
+rsa==4.9
+ruamel.yaml.clib==0.2.7
+ruamel.yaml==0.18.5
+s3transfer==0.7.0
+sacremoses==0.0.53
+safetensors==0.3.3
+sagemaker-headless-execution-driver==0.0.12
+sagemaker-jupyterlab-emr-extension==0.1.9
+sagemaker-jupyterlab-extension-common==0.1.9
+sagemaker-jupyterlab-extension==0.2.0
+sagemaker-kernel-wrapper==0.0.2
+sagemaker-studio-analytics-extension==0.0.21
+sagemaker-studio-sparkmagic-lib==0.1.4
+sagemaker==2.198.0
+sasl==0.3.1
+schema==0.7.5
+scikit-image==0.19.3
+scikit-learn==1.3.2
+scipy==1.11.4
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==1.38.0
+seqeval==1.2.2
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+simpervisor==1.0.0
+sip==6.7.12
+six==1.16.0
+smart-open==5.2.1
+smdebug-rulesconfig==1.0.1
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.2.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sparkmagic==0.21.0
+sqlalchemy==1.4.49
+srsly==2.4.8
+stack-data==0.6.2
+statsforecast==1.4.0
+statsmodels==0.14.0
+supervisor==4.2.5
+sympy==1.12
+tabulate==0.9.0
+tblib==1.7.0
+tenacity==8.2.3
+tensorboard-data-server==0.7.0
+tensorboard==2.12.3
+tensorflow-estimator==2.12.0
+tensorflow==2.12.1
+termcolor==2.3.0
+terminado==0.18.0
+text-unidecode==1.3
+textwrap3==0.9.2
+thinc==8.2.1
+threadpoolctl==3.2.0
+thrift-sasl==0.4.3
+thrift==0.19.0
+tifffile==2023.12.9
+tiktoken==0.5.2
+timm==0.9.12
+tinycss2==1.2.1
+tokenizers==0.13.3
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.12.3
+toolz==0.12.0
+torch==2.0.0.post200
+torchmetrics==1.0.3
+torchvision==0.15.2a0+072ec57
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.14.0
+transformers==4.31.0
+truststore==0.8.0
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+types-pytz==2023.3.1.1
+typing-extensions==4.5.0
+typing-inspect==0.9.0
+typing-utils==0.1.0
+typish==1.9.3
+tzdata==2023.3
+ujson==5.9.0
+unicodedata2==15.1.0
+uri-template==1.3.0
+urllib3==1.26.18
+wandb==0.16.1
+wasabi==1.1.2
+wcwidth==0.2.12
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.7.0
+werkzeug==3.0.1
+whatthepatch==1.0.5
+wheel==0.42.0
+widgetsnbextension==4.0.9
+window-ops==0.0.14
+wrapt==1.16.0
+xgboost==1.7.6
+xxhash==3.4.1
+yapf==0.40.1
+yarl==1.9.3
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.22.0

wandb/run-20240208_185311-5uym8l7w/files/wandb-metadata.json

@@ -0,0 +1,1029 @@
+{
+    "os": "Linux-4.14.336-253.554.amzn2.x86_64-x86_64-with-glibc2.35",
+    "python": "3.10.13",
+    "heartbeatAt": "2024-02-08T18:53:11.692683",
+    "startedAt": "2024-02-08T18:53:11.292051",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "/home/sagemaker-user/output-7b-26k-lora/../lora_finetuning_push_to_hub_save_local_latest.py",
+    "codePathLocal": null,
+    "host": "default",
+    "username": "sagemaker-user",
+    "executable": "/opt/conda/bin/python3",
+    "cpu_count": 96,
+    "cpu_count_logical": 192,
+    "cpu_freq": {
+        "current": 3259.3440989583337,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3299.457,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.806,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3205.787,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3178.002,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3200.706,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3196.209,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3188.711,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3189.097,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3206.992,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3200.764,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2810.939,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2847.942,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2923.114,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3028.23,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3012.472,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3044.914,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2953.37,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2957.586,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2984.294,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2961.352,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2901.559,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2801.726,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2985.17,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2963.11,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2912.001,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2965.712,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2966.821,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2871.172,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2974.758,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2989.099,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2948.999,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2895.266,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.988,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2924.435,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2919.839,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2875.943,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.697,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2805.016,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.583,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.604,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2673.256,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3296.503,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3139.11,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3137.942,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2833.969,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3153.277,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3178.769,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3207.604,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2892.532,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.772,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.641,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.096,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.515,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.26,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.084,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.903,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.866,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.276,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.704,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.342,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.795,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.923,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.013,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.43,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.564,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3300.54,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2747.263,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.353,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.896,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2533.725,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.656,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3293.031,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.834,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3024.556,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3067.379,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3010.826,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3101.81,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2973.599,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3061.27,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3291.322,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3017.723,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2660.496,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3004.775,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3021.086,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.592,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.589,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3019.568,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3029.623,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3080.312,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3066.263,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2998.37,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2949.133,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2964.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3222.788,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.63,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2916.281,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2825.282,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3038.106,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2895.235,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3092.874,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2924.994,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2913.404,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2935.638,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2583.261,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3101.162,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3063.704,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3093.23,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3095.386,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2925.773,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2920.019,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2916.15,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2944.025,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3259.667,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3049.572,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3263.675,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3074.497,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2923.985,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2910.425,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2812.861,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2874.988,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3120.953,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3124.25,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3113.753,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3119.282,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2982.281,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3048.291,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2987.986,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2733.968,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3274.202,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3120.154,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3122.388,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2592.46,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3121.448,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3085.363,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3176.23,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3098.413,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3131.838,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3297.418,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3144.573,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3142.177,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3135.089,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3124.315,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3206.745,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3197.608,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3271.659,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3055.483,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.813,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.316,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.471,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3275.344,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.318,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3272.185,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.032,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3273.055,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3277.573,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3274.44,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3275.925,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3279.092,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3275.089,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3277.671,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.135,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.31,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.038,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3218.557,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.859,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.545,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.843,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.687,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3053.229,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.26,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3059.862,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3090.937,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3094.897,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3083.774,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3027.722,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3303.02,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3069.951,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3049.694,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2814.624,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3097.913,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2788.423,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.195,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3069.533,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3074.679,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3066.308,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2598.471,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.109,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.455,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.945,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3298.926,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.003,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3299.535,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 32.0,
+            "used": 0.01256561279296875
+        }
+    },
+    "gpu": "NVIDIA A10G",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        },
+        {
+            "name": "NVIDIA A10G",
+            "memory_total": 24146608128
+        }
+    ],
+    "memory": {
+        "total": 747.9597625732422
+    }
+}

wandb/run-20240208_185311-5uym8l7w/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 17}}