8fa61cb158cb691aa1bb84da41ff32d779b7b4be6043cf0e368ee6c84931c0f0

experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-part1.ipynb

@@ -3,13 +3,13 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "f5ebb021",
+   "id": "ef458e0c",
    "metadata": {
     "papermill": {
-     "duration": 0.002819,
-     "end_time": "2023-09-29T04:50:11.669839",
+     "duration": 0.002614,
+     "end_time": "2023-09-29T05:06:25.725060",
      "exception": false,
-     "start_time": "2023-09-29T04:50:11.667020",
+     "start_time": "2023-09-29T05:06:25.722446",
      "status": "completed"
     },
     "tags": []
@@ -23,13 +23,13 @@
   {
    "attachments": {},
    "cell_type": "markdown",
-   "id": "6e6abc3f",
+   "id": "58eb3f3e",
    "metadata": {
     "papermill": {
-     "duration": 0.00214,
-     "end_time": "2023-09-29T04:50:11.676239",
+     "duration": 0.00201,
+     "end_time": "2023-09-29T05:06:25.730966",
      "exception": false,
-     "start_time": "2023-09-29T04:50:11.674099",
+     "start_time": "2023-09-29T05:06:25.728956",
      "status": "completed"
     },
     "tags": []
@@ -41,19 +41,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "3d2405bd",
+   "id": "e0abbad9",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:11.682830Z",
-     "iopub.status.busy": "2023-09-29T04:50:11.682139Z",
-     "iopub.status.idle": "2023-09-29T04:50:12.432460Z",
-     "shell.execute_reply": "2023-09-29T04:50:12.431486Z"
+     "iopub.execute_input": "2023-09-29T05:06:25.737449Z",
+     "iopub.status.busy": "2023-09-29T05:06:25.736495Z",
+     "iopub.status.idle": "2023-09-29T05:06:26.482958Z",
+     "shell.execute_reply": "2023-09-29T05:06:26.482054Z"
     },
     "papermill": {
-     "duration": 0.756299,
-     "end_time": "2023-09-29T04:50:12.434815",
+     "duration": 0.751859,
+     "end_time": "2023-09-29T05:06:26.485032",
      "exception": false,
-     "start_time": "2023-09-29T04:50:11.678516",
+     "start_time": "2023-09-29T05:06:25.733173",
      "status": "completed"
     },
     "tags": []
@@ -69,19 +69,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "id": "66fd5201",
+   "id": "42d56a7f",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:12.446546Z",
-     "iopub.status.busy": "2023-09-29T04:50:12.446098Z",
-     "iopub.status.idle": "2023-09-29T04:50:12.454394Z",
-     "shell.execute_reply": "2023-09-29T04:50:12.453644Z"
+     "iopub.execute_input": "2023-09-29T05:06:26.491452Z",
+     "iopub.status.busy": "2023-09-29T05:06:26.490928Z",
+     "iopub.status.idle": "2023-09-29T05:06:26.499148Z",
+     "shell.execute_reply": "2023-09-29T05:06:26.498384Z"
     },
     "papermill": {
-     "duration": 0.018125,
-     "end_time": "2023-09-29T04:50:12.456177",
+     "duration": 0.013307,
+     "end_time": "2023-09-29T05:06:26.500768",
      "exception": false,
-     "start_time": "2023-09-29T04:50:12.438052",
+     "start_time": "2023-09-29T05:06:26.487461",
      "status": "completed"
     },
     "tags": []
@@ -140,19 +140,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "id": "e0b56789",
+   "id": "5514ed91",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:12.464627Z",
-     "iopub.status.busy": "2023-09-29T04:50:12.464037Z",
-     "iopub.status.idle": "2023-09-29T04:50:42.488005Z",
-     "shell.execute_reply": "2023-09-29T04:50:42.486665Z"
+     "iopub.execute_input": "2023-09-29T05:06:26.507274Z",
+     "iopub.status.busy": "2023-09-29T05:06:26.506786Z",
+     "iopub.status.idle": "2023-09-29T05:06:55.991075Z",
+     "shell.execute_reply": "2023-09-29T05:06:55.990231Z"
     },
     "papermill": {
-     "duration": 30.031629,
-     "end_time": "2023-09-29T04:50:42.490859",
+     "duration": 29.490941,
+     "end_time": "2023-09-29T05:06:55.994238",
      "exception": false,
-     "start_time": "2023-09-29T04:50:12.459230",
+     "start_time": "2023-09-29T05:06:26.503297",
      "status": "completed"
     },
     "tags": []
@@ -162,14 +162,20 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[2023-09-29 04:50:16,856] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+      "[2023-09-29 05:06:30,625] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n",
       "---- Initializing model ----\r\n",
       "No of layers: 6\r\n",
       "Embedding size: 2048\r\n",
@@ -228,42 +234,42 @@
      "output_type": "stream",
      "text": [
       "2048  2048  0    blocks.0.ffn.receptance.weight\r\n",
-      "2048  7168  0    blocks.0.ffn.value.weight\r\n"
+      "2048  7168  0    blocks.0.ffn.value.weight\r\n",
+      "2048  2048  1.0  blocks.1.att.gate.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.1.att.gate.weight\r\n"
+      "2048  2048  1.0  blocks.1.att.receptance.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.1.att.receptance.weight\r\n"
+      "2048  2048  1.0  blocks.1.att.key.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.1.att.key.weight\r\n"
+      "2048  2048  1.0  blocks.1.att.value.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.1.att.value.weight\r\n"
+      "2048  2048  0    blocks.1.att.output.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  0    blocks.1.att.output.weight\r\n",
       "7168  2048  1.0  blocks.1.ffn.key.weight\r\n"
      ]
     },
@@ -272,42 +278,42 @@
      "output_type": "stream",
      "text": [
       "2048  2048  0    blocks.1.ffn.receptance.weight\r\n",
-      "2048  7168  0    blocks.1.ffn.value.weight\r\n",
-      "2048  2048  1.0  blocks.2.att.gate.weight\r\n"
+      "2048  7168  0    blocks.1.ffn.value.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.2.att.receptance.weight\r\n"
+      "2048  2048  1.0  blocks.2.att.gate.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.2.att.key.weight\r\n"
+      "2048  2048  1.0  blocks.2.att.receptance.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  1.0  blocks.2.att.value.weight\r\n"
+      "2048  2048  1.0  blocks.2.att.key.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  0    blocks.2.att.output.weight\r\n"
+      "2048  2048  1.0  blocks.2.att.value.weight\r\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "2048  2048  0    blocks.2.att.output.weight\r\n",
       "7168  2048  1.0  blocks.2.ffn.key.weight\r\n"
      ]
     },
@@ -360,13 +366,7 @@
      "output_type": "stream",
      "text": [
       "2048  2048  0    blocks.3.ffn.receptance.weight\r\n",
-      "2048  7168  0    blocks.3.ffn.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "2048  7168  0    blocks.3.ffn.value.weight\r\n",
       "2048  2048  1.0  blocks.4.att.gate.weight\r\n"
      ]
     },
@@ -404,13 +404,7 @@
      "output_type": "stream",
      "text": [
       "2048  2048  0    blocks.4.ffn.receptance.weight\r\n",
-      "2048  7168  0    blocks.4.ffn.value.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "2048  7168  0    blocks.4.ffn.value.weight\r\n",
       "2048  2048  1.0  blocks.5.att.gate.weight\r\n"
      ]
     },
@@ -439,13 +433,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "2048  2048  0    blocks.5.att.output.weight\r\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
+      "2048  2048  0    blocks.5.att.output.weight\r\n",
       "7168  2048  1.0  blocks.5.ffn.key.weight\r\n"
      ]
     },
@@ -471,13 +459,13 @@
   },
   {
    "cell_type": "markdown",
-   "id": "e3057f8b",
+   "id": "8afd9e50",
    "metadata": {
     "papermill": {
-     "duration": 0.006306,
-     "end_time": "2023-09-29T04:50:42.503924",
+     "duration": 0.005752,
+     "end_time": "2023-09-29T05:06:56.006385",
      "exception": false,
-     "start_time": "2023-09-29T04:50:42.497618",
+     "start_time": "2023-09-29T05:06:56.000633",
      "status": "completed"
     },
     "tags": []
@@ -489,19 +477,19 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "c06c6ad2",
+   "id": "ff78d2bd",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:42.519776Z",
-     "iopub.status.busy": "2023-09-29T04:50:42.518734Z",
-     "iopub.status.idle": "2023-09-29T04:50:47.537394Z",
-     "shell.execute_reply": "2023-09-29T04:50:47.535673Z"
+     "iopub.execute_input": "2023-09-29T05:06:56.020959Z",
+     "iopub.status.busy": "2023-09-29T05:06:56.020447Z",
+     "iopub.status.idle": "2023-09-29T05:07:01.579575Z",
+     "shell.execute_reply": "2023-09-29T05:07:01.578476Z"
     },
     "papermill": {
-     "duration": 5.029265,
-     "end_time": "2023-09-29T04:50:47.539698",
+     "duration": 5.569483,
+     "end_time": "2023-09-29T05:07:01.582319",
      "exception": false,
-     "start_time": "2023-09-29T04:50:42.510433",
+     "start_time": "2023-09-29T05:06:56.012836",
      "status": "completed"
     },
     "tags": []
@@ -527,19 +515,19 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "4cc7e34f",
+   "id": "f656d56b",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:47.552392Z",
-     "iopub.status.busy": "2023-09-29T04:50:47.551853Z",
-     "iopub.status.idle": "2023-09-29T04:50:47.806392Z",
-     "shell.execute_reply": "2023-09-29T04:50:47.805379Z"
+     "iopub.execute_input": "2023-09-29T05:07:01.598719Z",
+     "iopub.status.busy": "2023-09-29T05:07:01.597947Z",
+     "iopub.status.idle": "2023-09-29T05:07:01.851778Z",
+     "shell.execute_reply": "2023-09-29T05:07:01.850738Z"
     },
     "papermill": {
-     "duration": 0.264553,
-     "end_time": "2023-09-29T04:50:47.809133",
+     "duration": 0.265316,
+     "end_time": "2023-09-29T05:07:01.854564",
      "exception": false,
-     "start_time": "2023-09-29T04:50:47.544580",
+     "start_time": "2023-09-29T05:07:01.589248",
      "status": "completed"
     },
     "tags": []
@@ -549,17 +537,16 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/usr/bin/sh: 1: cd: can't cd to {TRAINER_DIR}\r\n"
+      "/usr/bin/sh: 1: python: not found\r\n"
      ]
     }
    ],
    "source": [
     "# Start the foundation model training\n",
     "!cd \"{TRAINER_DIR}\" && \\\n",
-    "    export RWKV_WAVENET_LAYERS=\"{RWKV_WAVENET_LAYERS}\" && \\\n",
     "    export WANDB_MODE=\"{WANDB_MODE}\" && \\\n",
     "    python lightning_trainer.py fit \\\n",
-    "        -c \"{NOTEBOOK_DIR}/v5base-enwiki-4k.yaml\" \\\n",
+    "        -c \"{NOTEBOOK_DIR}/v5base-enwiki-4k-part1.yaml\" \\\n",
     "        --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Enwiki-4k Part 1 (train-ctx=4k, {DEEPSPEED_STRAT})\" \\\n",
     "        --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n",
     "        --trainer.devices=\"{GPU_DEVICES}\" \\\n",
@@ -572,19 +559,19 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "0b3b8134",
+   "id": "c7b46f94",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:47.825099Z",
-     "iopub.status.busy": "2023-09-29T04:50:47.824495Z",
-     "iopub.status.idle": "2023-09-29T04:50:48.327589Z",
-     "shell.execute_reply": "2023-09-29T04:50:48.326466Z"
+     "iopub.execute_input": "2023-09-29T05:07:01.871225Z",
+     "iopub.status.busy": "2023-09-29T05:07:01.870345Z",
+     "iopub.status.idle": "2023-09-29T05:07:02.373808Z",
+     "shell.execute_reply": "2023-09-29T05:07:02.372753Z"
     },
     "papermill": {
-     "duration": 0.514109,
-     "end_time": "2023-09-29T04:50:48.330177",
+     "duration": 0.51526,
+     "end_time": "2023-09-29T05:07:02.376685",
      "exception": false,
-     "start_time": "2023-09-29T04:50:47.816068",
+     "start_time": "2023-09-29T05:07:01.861425",
      "status": "completed"
     },
     "tags": []
@@ -615,19 +602,19 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "92869fb9",
+   "id": "9f558c57",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-09-29T04:50:48.346924Z",
-     "iopub.status.busy": "2023-09-29T04:50:48.346311Z",
-     "iopub.status.idle": "2023-09-29T04:50:48.600443Z",
-     "shell.execute_reply": "2023-09-29T04:50:48.599423Z"
+     "iopub.execute_input": "2023-09-29T05:07:02.393471Z",
+     "iopub.status.busy": "2023-09-29T05:07:02.392695Z",
+     "iopub.status.idle": "2023-09-29T05:07:08.804315Z",
+     "shell.execute_reply": "2023-09-29T05:07:08.803244Z"
     },
     "papermill": {
-     "duration": 0.26565,
-     "end_time": "2023-09-29T04:50:48.603118",
+     "duration": 6.42299,
+     "end_time": "2023-09-29T05:07:08.806769",
      "exception": false,
-     "start_time": "2023-09-29T04:50:48.337468",
+     "start_time": "2023-09-29T05:07:02.383779",
      "status": "completed"
     },
     "tags": []
@@ -637,14 +624,28 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/usr/bin/sh: 1: cd: can't cd to {INFERENCE_DIR}\r\n"
+      "[2023-09-29 05:07:06,749] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n",
+      "Traceback (most recent call last):\r\n",
+      "  File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/dragon_test.py\", line 52, in <module>\r\n",
+      "    model = SimpleRWKV(MODEL_PATH, device=DEVICE)\r\n",
+      "  File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/src/model.py\", line 1420, in __init__\r\n",
+      "    self.model = RWKV(**model_config)\r\n",
+      "  File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/src/model.py\", line 566, in __init__\r\n",
+      "    raise ValueError(f\"load_model file '{load_model}' does not exist\")\r\n",
+      "ValueError: load_model file '../model/v5-L6-D2048-E0_01-enwiki-4k-p1.pth' does not exist\r\n"
      ]
     }
    ],
    "source": [
     "# # Lets do a quick dragon prompt validation\n",
     "!cd \"{INFERENCE_DIR}\" && \\\n",
-    "    export RWKV_WAVENET_LAYERS=\"{RWKV_WAVENET_LAYERS}\" && \\\n",
     "    python3 dragon_test.py \"../model/{FILENAME_PREFIX}-enwiki-4k-p1.pth\" \"cuda fp32\""
    ]
   }
@@ -669,14 +670,14 @@
   },
   "papermill": {
    "default_parameters": {},
-   "duration": 38.643439,
-   "end_time": "2023-09-29T04:50:49.032467",
+   "duration": 44.644446,
+   "end_time": "2023-09-29T05:07:09.133994",
    "environment_variables": {},
    "exception": null,
    "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-part1.ipynb",
    "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/multi-size-train/v5-L6-D2048-part1.ipynb",
    "parameters": {},
-   "start_time": "2023-09-29T04:50:10.389028",
+   "start_time": "2023-09-29T05:06:24.489548",
    "version": "2.4.0"
   }
  },