715a8b6edb467dc7d61ce79795e6fc7d3f73b519493e4f2d7c06cfcc7106186f

experiment/rwkv-x-exp/v5-headsize2x/hello-world.ipynb

@@ -3,19 +3,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "fbe2a012",
+   "id": "5a2bbd3c",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2023-08-24T03:57:18.245198Z",
-     "iopub.status.busy": "2023-08-24T03:57:18.244865Z",
-     "iopub.status.idle": "2023-08-24T03:57:18.489940Z",
-     "shell.execute_reply": "2023-08-24T03:57:18.489209Z"
+     "iopub.execute_input": "2023-08-24T04:44:25.040059Z",
+     "iopub.status.busy": "2023-08-24T04:44:25.039841Z",
+     "iopub.status.idle": "2023-08-24T04:44:25.284633Z",
+     "shell.execute_reply": "2023-08-24T04:44:25.283834Z"
     },
     "papermill": {
-     "duration": 0.249423,
-     "end_time": "2023-08-24T03:57:18.491807",
+     "duration": 0.248685,
+     "end_time": "2023-08-24T04:44:25.286190",
      "exception": false,
-     "start_time": "2023-08-24T03:57:18.242384",
+     "start_time": "2023-08-24T04:44:25.037505",
      "status": "completed"
     },
     "tags": []
@@ -32,6 +32,123 @@
    "source": [
     "!echo \"Hello world\" # this file is for debugging the runner"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5b3eecbc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-08-24T04:44:25.289218Z",
+     "iopub.status.busy": "2023-08-24T04:44:25.289031Z",
+     "iopub.status.idle": "2023-08-24T04:44:25.295282Z",
+     "shell.execute_reply": "2023-08-24T04:44:25.294543Z"
+    },
+    "papermill": {
+     "duration": 0.009124,
+     "end_time": "2023-08-24T04:44:25.296411",
+     "exception": false,
+     "start_time": "2023-08-24T04:44:25.287287",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DEEPSPEED_STRAT: deepspeed_stage_2_offload\n",
+      "ENABLE_WANDB: True\n",
+      "GPU_DEVICES: auto\n",
+      "NOTEBOOK_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-headsize2x\n",
+      "INFERENCE_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5headsize2x\n",
+      "TRAINER_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5headsize2x\n",
+      "PROJECT_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer\n"
+     ]
+    }
+   ],
+   "source": [
+    "DEEPSPEED_STRAT=\"deepspeed_stage_2_offload\"\n",
+    "GPU_DEVICES=\"auto\"\n",
+    "ENABLE_WANDB=True\n",
+    "\n",
+    "RWKV_WAVENET_LAYERS=1\n",
+    "\n",
+    "EMBED_SCALE=0.1\n",
+    "EMBED_SCALE_LABEL=str(EMBED_SCALE).replace(\".\", \"_\")\n",
+    "\n",
+    "LAYER_COUNT=6\n",
+    "EMBED_DIM=4096\n",
+    "\n",
+    "WANDB_PREFIX=f\"v5-hs2x-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n",
+    "FILENAME_PREFIX=f\"v5-hs2x-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\"\n",
+    "\n",
+    "print(\"DEEPSPEED_STRAT:\", DEEPSPEED_STRAT)\n",
+    "print(\"ENABLE_WANDB:\", ENABLE_WANDB)\n",
+    "print(\"GPU_DEVICES:\", GPU_DEVICES)\n",
+    "\n",
+    "if ENABLE_WANDB:\n",
+    "    WANDB_MODE=\"online\"\n",
+    "else:\n",
+    "    WANDB_MODE=\"disabled\"\n",
+    "\n",
+    "# Computing the notebook, and various paths\n",
+    "import os\n",
+    "NOTEBOOK_DIR=os.path.dirname(os.path.abspath(\"__file__\"))\n",
+    "PROJECT_DIR=os.path.abspath(os.path.join(NOTEBOOK_DIR, \"../../../../\"))\n",
+    "TRAINER_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5headsize2x/\"))\n",
+    "INFERENCE_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5headsize2x/\"))\n",
+    "\n",
+    "print(\"NOTEBOOK_DIR:\", NOTEBOOK_DIR)\n",
+    "print(\"INFERENCE_DIR:\", INFERENCE_DIR)\n",
+    "print(\"TRAINER_DIR:\", TRAINER_DIR)\n",
+    "print(\"PROJECT_DIR:\", PROJECT_DIR)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "39c31880",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2023-08-24T04:44:25.299318Z",
+     "iopub.status.busy": "2023-08-24T04:44:25.299143Z",
+     "iopub.status.idle": "2023-08-24T04:44:25.772732Z",
+     "shell.execute_reply": "2023-08-24T04:44:25.771978Z"
+    },
+    "papermill": {
+     "duration": 0.476676,
+     "end_time": "2023-08-24T04:44:25.774132",
+     "exception": false,
+     "start_time": "2023-08-24T04:44:25.297456",
+     "status": "completed"
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5headsize2x\r\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 4.0K\r\n",
+      "drwxr-xr-x  2 root root   10 Aug 24 04:44 .\r\n",
+      "drwxr-xr-x 19 root root 4.0K Aug 24 04:44 ..\r\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Get the current model listing\n",
+    "!cd \"{TRAINER_DIR}\" && pwd\n",
+    "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/\""
+   ]
   }
  ],
  "metadata": {
@@ -49,14 +166,14 @@
   },
   "papermill": {
    "default_parameters": {},
-   "duration": 1.423604,
-   "end_time": "2023-08-24T03:57:18.709651",
+   "duration": 1.911028,
+   "end_time": "2023-08-24T04:44:25.992833",
    "environment_variables": {},
    "exception": null,
    "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-headsize2x/hello-world.ipynb",
    "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/v5-headsize2x/hello-world.ipynb",
    "parameters": {},
-   "start_time": "2023-08-24T03:57:17.286047",
+   "start_time": "2023-08-24T04:44:24.081805",
    "version": "2.4.0"
   }
  },