1.5b and 3b runs

RWKV · Jan 23, 2024 · dc99b9b · dc99b9b
1 parent 54c816a
commit dc99b9b
Show file tree

Hide file tree

Showing 2 changed files with 2,596 additions and 6 deletions.
diff --git a/notebook/rwkv-x-exp/v5-exp/memory-test/World-1B5-mem-finetune.ipynb b/notebook/rwkv-x-exp/v5-exp/memory-test/World-1B5-mem-finetune.ipynb
@@ -176,7 +176,7 @@
     "tags": []
    },
    "source": [
-    "## Finetune 1 (0 -> 4k) : Dataset preperation\n",
+    "## Finetune 1 (0 -> 2x2k) : Dataset preperation\n",
     "\n",
     "Stage 1, handles total context size of 2048. Meaning it will be tuned for memory task of 1 to approximately 1024 tokens of size."
    ]
@@ -328,7 +328,7 @@
     "tags": []
    },
    "source": [
-    "## Finetune 1 (0 -> 4k) : The actual tune!"
+    "## Finetune 1 (0 -> 2x2k) : The actual tune!"
    ]
   },
   {
@@ -429,7 +429,7 @@
    "id": "8259d9b5",
    "metadata": {},
    "source": [
-    "## Finetune 1 (0 -> 2*2k) : The actual tune!"
+    "## Finetune 1 (0 -> 2x2k) : The actual tune!"
    ]
   },
   {
@@ -485,7 +485,7 @@
    "id": "6fe5d71b",
    "metadata": {},
    "source": [
-    "## Finetune 2 (0 -> 2*4k) : Dataset preperation\n",
+    "## Finetune 2 (2x2k -> 2x4k) : Dataset preperation\n",
     "\n",
     "Stage 2, handles total context size of 8k. Meaning it will be tuned for memory task of approximately 4k tokens of size."
    ]
@@ -2272,7 +2272,7 @@
     "#\n",
     "# Ramping up the 3000+ - 400 words dataset\n",
     "# \n",
-    "for i in {3000..6000..25} \n",
+    "for i in {3025..6000..25} \n",
     "do\n",
     "    python ./memory_script/gen_limited_prompt_completion_jsonl.py ./dataset/gen-word-$i-count.jsonl $i 100 & \n",
     "    python ./memory_script/shuffle_limited_prompt_completion_jsonl.py ./dataset/shuffle-word-$i-count.jsonl $i 100 & \n",
@@ -2325,7 +2325,7 @@
    "id": "60244472",
    "metadata": {},
    "source": [
-    "## Finetune 1 (0 -> 2*2k) : The actual tune!"
+    "## Finetune 2 (2x2k -> 2x4k) : The actual tune!"
    ]
   },
   {