diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..162014b7ab83fedc0514f5fed8c35b809dbe2716 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,55 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_11 filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_17 filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_6 filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_9 filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
+moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl filter=lfs diff=lfs merge=lfs -text
diff --git a/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..026045b7ad4ffbdcfef669cc4c93c932b9f5a397
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..ac7bba96bc4e16547a8671ff43cb2d5bb238c508
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb85d40c345f9a9af446393b4f586bf2ce65fa452d9dcc4c87d3f8edb5524dbe
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..1a4e8b0664b273a8a8788ff1ae2d8b0e34c21fb1
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7182807573412037
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..8301f6fd925a0e25f69b9818aa62cd20c4121788
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..9de46d2807d95fb4c8fb40399dc2847984704be4
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20bed1e67c3ee1fe4f7cc7ef473860255aa95583f509134bbcc77f4fb0cdf063
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..ce49737401460cd9d6471d561863c3f33740e447
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7568720467744474
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..070d8122729717df61a780a181bd22c8ffaf92d4
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..e638d10a618c74ad6bac79b62ad0a7e1c7232751
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cfc61d7231941624afb21ec3a335a567f6354bd3baaf614d859829fe7d5cbaa
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..8528bb2e32666cfc469c11896839e82dd06abce1
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.748058017777638
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..5aed2f73ff8f0f46570a53e782c1ae7d3af83a59
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..58e2b0533f53b96730fb3348292748f909e6cc07
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3068e0abe0fe9d83bd07bbc7064606e6032ddb5a4902e3421066a87fb0011513
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..bfeffe692d053de802f6fb5cab56ad87e8714d9a
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7405886863413804
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..36eecb3f313a755b7c964c84129096f01eac78fe
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..611151dc158dd0538beaf61f6da2c28d7144f8ac
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2af635c227579f92a27d37848a14c1b05b6d94db24a3f2b5626adfa1cd07a6d
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..9eef75c87475ef5947c03bb3ca11dd7b227a91b6
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7240940497555308
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..66303fedf3ac0573a1b655c4ead9c8a3b38ed778
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..51d04bda00528c2e25713fb68aa6800e4c3895db
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e77bab9ec7867aa7368ba578158e213993b9db288b38b91ec92df78e528cd53
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..fd50a1e9ec4a83f97799b19ceb2b0d5403099510
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7124260153281037
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..fa2a5079d907307d543a45e74b10f5fcde339edc
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..087af7214c2e0e3b90e2d9452398e81a4773441a
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d7df88eb6c2c54408411dde7b833169ab32bff6afcbdca26472cf1c504f5d3d
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..89b362ea762f938f9dda19de071300e58c8b752d
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7323668953459915
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..448505e4852c740c8a29dd9ca48d1a930761b82e
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..c31ee231aa8983db8a14640f363fdcb965e3c504
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f16e465cd271ee72ce95d1e21ea1c32ef65b5296ff65557e4d0a029be41a4a93
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..7a42c80060b87fdfdcf3f30773bc68d71dc939af
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7322201105149965
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..1eb7d28ca1868e76c866fd842db0b0b83a820ba3
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..c60790874fabd89a540db849e9dd4ba2cba3669b
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bfbfb4bfc22ca4587d1ac4e78e53b71cac7352de8c03d30a87165e5c2b2a8e2
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..d29c7f74b67e6e142fe696f7f3fcdecb4e2ace9d
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.731212756898174
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..570239846470d2aa3f405c341edcfcb6f1d39855
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..4c29fadeb38bba534272ca138c9f77f80dcde806
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a1ddb9e1cafccbf2b8eab8b81558f59126005230c99253426e14aca5f72744d
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..a31422f65a5300a2fbe30ddf5591409581db4daf
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7087740436002287
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..d5152562cf2b3f9f88748e44a5ee37e209ba9745
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..99e56c50cef0d3ec74e0ed189c17d38670739c73
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a28c5d6ec9002cf88f6795fbaee9c77721eeae13aaf3abf283443db176c62c8f
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..48bb0fa1309f82cfc91b7fcdd557e4d7c158ff48
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7103819609011953
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..8459cc8362e77a1cbf0e82c3776ee9595d38c002
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..2120d0a9842013b5e24dc113749c0688f323345a
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91801389212fdef862996cb249248b4f3b01e407d3effcb99ff954ce22a1c548
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..681c3bec81491ab3401a88c24a2deed8a0b8a77c
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6948271535597236
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..a4983d42a60831355a269daa3dfa13023f340183
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..66bbcd5646a83a3a2d7421e351cf96414b09cd81
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4297fe8c193d47f269e3d0e5cdb69163defd1a14c5503f8c773e93ef179d7a1d
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..7309462bd4f5641c5eb56bba8326f9f16e40288e
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7546966849540423
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..d01767d09883adba760260af211c0208ad9cbc6b
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..bed29e340d7c5cfcfb529f6de8013a41e1d6b4f8
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fa933e0a510a761361b52ad631b867692ca6159b17d3fad94b3791ddf2fc87d
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..12208507463da28bb6bea008c70107cad2dee84f
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7048745886978236
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..1a352aafd46e9f024fda38bdeedbac30eae3222f
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..33e8221ff7458d92828c07056418fbe59ae56e90
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d918044c9d0aff83da39d103a70ee4623c495a54f41ff6c537b1d2476b0b9fe
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..c69ce1b3230846528804341489b5f66b67c5a4bd
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6884838024893459
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..39ad685a75ab651f9d84ac3be19e04cb77022ac5
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..8d2b6f2ab78545f4c989771728c847edf988aa8d
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed3070812d08c2f1b0a60bd5e6caf40010e6fd11485ad450e30b9ddbae727a8b
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..60ebcc81dc54116ca2232b42bb1e1b55075085e9
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.695699293593393
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..81dec2433f0f4653c8ec64de62be65d18a6d77cb
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..c3e01598df3e0f13557aac6d74748af84a597f5c
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3300d93f20e16f2d0a8050eebffdd46aa99be4ee952a26cfa2d4e535366f8fa
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..e5939d082fdb5495128e2dc96f6c178d81174c85
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.8097830117537499
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..ce34859fbc3338d6ee90ff29be178e118037fb06
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..37426c06e1ba9bbe490d6f18306163a68a9ef078
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09bde66667b9530ac83e5f569dfb85f84b120a5b2f8959681ca3bebef111a165
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..c316e355d1739af5ea676dfa6790ac364a575eff
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7244562435097778
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..1b7f30b41b91fa0651f738b60d14d1a07de4c10b
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..e565aa02ee9bcf6ebcbd071827557edd123423ce
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e942b7587c95d8590d1c5e2b55e2cf70d22acb90089580787f6ea351645fb16f
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..f0f0acc4a84cf5ee472e8a782a65378100993e83
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.8235359317051338
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..425bb57062ea846b3bb47a0a49fa975f3ddb4da3
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..10a60ce3afff1bd5d350d3d6d3053ec03b4b53a5
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e479d5e85fdaabf2880975a4eed2f5042d3b4c378e5f0c47df7ccc02357c5a00
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..543e69d99247febda7fda120d927a5b5af3e2dda
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.8073691867087461
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..0cc8e6b7762ba0d70dc3db5081bfd3868f75cf67
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..e119889382974317b5c673e8633acc7f51173807
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:162130e743a41003b4065fff2e2b0ee7b0259d490cca38420d311a6498acfb20
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..dd4f6303e58c1e3bf2403dde1f814cdf0c0fb29f
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.807624740417183
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..9cbf797c178e5b4f8a0b09bf735a0350dfe75e52
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..a5e2ff765ee8a4d2bf416cd7d6ca2227cee7b539
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8be7c3d4fe51edc142bd2ba6dea9580c816895e2142749c1b8ef57514c607062
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..12b328951e32d8cdf977a32678da991dc2408ad7
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.8125181665237419
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..36bc1e00c1cb1bbd83f59cbd997a8ba01457d718
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..93696fa78ebbf0d80f57590b555f989a273f0daf
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7b14e746133ede31536ca957d2550156d21ca78f7eb59d77b20c5f67cb0215a
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..4b2b10950ba111c26bb0dec090833a02e341be08
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.8023329924487979
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..51810c13859f4321ff814406d8ace69a3e380880
Binary files /dev/null and b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..0fe0ba22e7b5d3fd1835e2d5b6f7fc55518ff040
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61beccb145df58cb9c46c63856b900c25a6f7c26dc6a426a8dc03b302ef156af
+size 3147751
diff --git a/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..bd273d169a272433375643cba5f3aeb0b7390060
--- /dev/null
+++ b/moe-3b-qdp/param_split/decoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7784156091534002
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..8eaf54d91f09eb6fd251e6c6e57d8a41fe7a64ca
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..e46d8926669b9d391f3731b9d65b981245a60022
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffb8fd78a14bf36d3964641d5d1103464dbb53bab06a906132eb69a7eb631565
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..674c717f24a15106fede48734b09179a97897220
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.0.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6367519461039036
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..f0562b65c329737cd96cd8ff274636fde17c0569
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..a63762bc8a867089d8259d713f131bc2e5601a3b
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47d6ce6cd3226c0ab3b86e20f800ef229f79eaa9ce3f77a544627218ebff26f2
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..ea8426616dc2926163ef649186c5f494c27dabff
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.1.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6856357720194128
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..c099e22409e72f7a1be35b052b6d341615bc4b42
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..77e4b6685cb68cfbe2343488d1724d1c6c209b61
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c633b45773d34c6695f862639a395366a70be2f0494b80ce7aec9019e2ce1a
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..b698a42eb2d81e5f7a4738a11a6d99c5a867a808
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.10.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6763765865837283
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..483af570ab12e41b18596a83db005b15ddf17863
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..085a6f91af05292f00e2a17fbdb1c293c65ec62d
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a66f40995bf286dd9e7b7402beec80b80dfe35fa6a00d8f4f879fb8423f8b14a
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..a10599d0d35007e1cb621831cc2179f24aa3a7f3
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.11.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6857803520204263
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..bf4ace8b64129c0521e74b56aee239e4d48d7735
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..122aee76f71b4dd72edaf2f0dba88686b2c5828d
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b30fa561fd732c837e0977e778e2a281742d167ecf9603e1387d52cead955c
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..d6b30e0c080e955d079ceafe1419c08e23b04274
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.12.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6834476773878034
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..c26c6d8d7c9a0069338ce751121ee8ca69c77fa1
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..3ac1c7b54f7ee35ab5005136eed7f7c8ebbdb79f
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bf71d5a6699b159ab01681330bc7ed6baed7f88ceb41f06b84b2ba29521f58d
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..8afd6939d6f83ffb08e992a4fde478b48fc14a03
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.13.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.69375418545075
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..9cab4401af83a3424c415ed9a152179db05686ae
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..b404e2714eeece9e42cd7eee6f404bcda0a3d200
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12f68d2e945942de7b6a2da59e0a6ba99183dcb079bde9fe0b471c3f7fdb2d7f
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..4fb751a935f810a5a342bdf6a6cd04b2ee6f878c
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.14.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7001958402549855
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..b3862809e658334cdbf3f1c5d57ad1ab569ea601
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..84584445df2d078c6f2c53205155f0c0cbfec0f2
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24fb9ddca38de932eb874feb8a4449ea28f07557f99a2577c9b695cb4cecde8
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..ce49eecdd56b729840e5e531caf0f9f85468b7cd
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.15.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6989576401289532
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..2f01c996c1637a22b2a97ede61e8419055306608
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..b514a2b7646470d286d530ffe262a3e854998978
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5952b16dc9f340266f8947e078a2cfeb632674997295040a5a2a1bf97d05e097
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..d2924ea80782018dda97ae7192f42beb53e61599
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.16.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7052567585434403
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..f1c36bb6c3bd5fcb5ea839cbfd103f2eb356d10e
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..ccc63e25c87126e379653ebd16453c44a6e71830
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c93f74852261e0ee14d290543b549fe9f4e11a7063556618ec18ffa5adace73
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..4c0be85f79c6d66aa8dac7c10234241286cadd96
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.17.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.7007782969437482
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..4c42b53d87dcb64364cfc900df1245b840770cbc
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..cc7914f8db2756628d0b60eded41b547b113d062
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e0bd311c54e56749ba46cd5e8d15c4696d963d5429489a707b1ef1519c30c00
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..9500941e6b401d79fa22198d9a5272680ce2ccab
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.18.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.693019934462939
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..364ea7cd592c46ec1c329a301bb9aa5669e0e90c
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..91cbe41e836034de5d6682b2f9d3aa5984b82332
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178b25da617007fa38b47dd77903f811d025df2681116cb5fdd4eaa3e8719afb
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..78eeb3de4af20b90ac77c551bd7e0feb02d769e2
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.19.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6799446178920192
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..892b2d39ef1c332c78e438aac5d124fc01b754c3
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..8f383630405ce9d6cd7f38170882239123eed25e
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5755f2d13944a94d3ada02c80e3340267cb8dc09a5aa9ac7585777d17b4941e9
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..6aa53db8e985c5486e5b7aa1c4c60bba41b21f2b
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.2.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6298436708404194
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..fe0f836a6a3d959aa16645e1440aa48bad6a62bc
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..b2cec45ef9893e16c065bbb6fce0166ff9fedf9c
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eb941cbfd0411c436d35556bb6e8eda946f293e858b75a20c8a012beb7739d7
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..2d2efe3cfa9b8fe567fed28ceb9bee8248adc95a
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.20.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6856017158695018
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..33be08a9be2a97e868757467c21d3c2d2b2b6b27
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..61e387d08492b869abc8bd1f0b962c4d2f42c198
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f5cd87e3ca8bb8ce3ab09426f0fca2c85c20435e6208c4c5537dc81fcd352b9
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..b0f3ea008de45749474fbe1a0295d621c88f0dda
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.21.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6948824230300396
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..60c4093df19107ce93f6dda9b9384e52188abf45
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..9ec471e701bb1a46b51ab9e5fa9d81efcad87aa8
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28c086be442ea282cb328e70fd0603793b13ea9fd959b2201249a5c3e6475970
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..d320bf866fd75c64a248569c1d635a5e7e004d83
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.22.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6678603309071452
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..82ce7aa92411558a03e9ea12675684c691dc79d4
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..a76457a4c85ad0a2172c46f9406f7ec58e5a69e0
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1484c5428720d9d5544a6bc4a6cf9250e1b4ed0fc60e4e6c8e1c1afd7571464f
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..4468316fd274f62526f760c65d098bc03a979b18
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.23.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6270927490403381
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..9ae9679fbe6625bdf85bcfae1a15cb378ec2b6c4
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..ca44cc2989af6f9648acd64a536a61ef9caa6cf1
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53f1fac65e8b790f0509951b596159c34cd9a71469569da721c0d9a601f77ba2
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..78edaf9c9cfba60ce4b81e27d853b22b917495ad
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.3.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6059153248752451
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..e346e7ef8f77fed4986f5ddf436fd027687f2ee5
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..e1f0be2488a6396432c1a3f328973e5ebe8a1bfa
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b64eb52aab8f14c3e17c2317fadbe2a691bcd4b0a01cb977e077679873409fd
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..a95bf2268d9a25e2007fa3b5d4e58a8332d5b6a7
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.4.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.5998099094190861
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..e5902ae3c6bfd6708404069e88b6c00a7b7a21c2
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..ec2581a0ddc45d1f31cee63fa676f72c4dcbb693
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:250edb34ab2dc8a65e5b4e6149a146e1354021d0edd95dc8fd4eec0d19646ba4
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..19372d00ba84f8eed4228b32a027aeb22c857485
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.5.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6168540446761476
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..8770961428fc38535448a3590b4aad325220821a
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..ac8c1e36d350c02f3ef0b0678449fca0a8a4c7fd
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e493067223cbb14a2569122b50ce7753c04bafb9b380ce9b7cb934acfd5c2269
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_11 b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_11
new file mode 100644
index 0000000000000000000000000000000000000000..73efc9489493ebaf58317ef0fd660d957d522cad
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_11
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eee278765dd5c5d9ccb5b7e49bb50f09c0f8691b5e1f1a1ed9e638cc91a757ad
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_17 b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_17
new file mode 100644
index 0000000000000000000000000000000000000000..f5bb1572d1ed2f1ab46ba2005f2f162a8c0c993a
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_17
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5613a89d821ddd9f8711caccf6d7f9b01518fe7aa8678f287b3655b17d58ef
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_6 b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_6
new file mode 100644
index 0000000000000000000000000000000000000000..5003d3545ee4b5d9bab4b853e1392b389cb95714
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_6
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d25d003e9728716c9dc9a7ec07efbe5f669812c9f12ee2ecdc60c1af04ff4efe
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_9 b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_9
new file mode 100644
index 0000000000000000000000000000000000000000..bc7e6969d8fa9ca2bd775d53ae176c622d4b6d3a
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_9
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a92f3aafc1841e2955de41525a1738822e47ecfa4373ae626f145aaede58d4ae
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..0adec30008483a6350bd6703f5d7c32e59143742
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.6.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6420725304124393
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..e7be4cc94c10f7edbbdcda82c79ef3240930216c
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..89bccca0ade9fd55e219f12fc02c7280584c7df0
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:117b20372bfb5d2e81b6c75f2012d36674231c6585abec5818be7786159124ce
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..a6a10ba44f4bfe112f85969f64471eac08588c41
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.7.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.668471283671508
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..daf7103b58a240dfac474e80bdabb785c1f3fc4f
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..930cb4c3ab48e616f4df87c31944feca99c4e8f7
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45e0499bab78b7a003f62a2f7c89f8d00158e1d906367e8c8a148e85580abaf6
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..522b52c7137481934cc8a0c242c9953378f69067
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.8.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.6567293316317655
\ No newline at end of file
diff --git a/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight
new file mode 100644
index 0000000000000000000000000000000000000000..2bee00d0b2ace433b9cbfed99e0f3502c28aa349
Binary files /dev/null and b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight differ
diff --git a/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl
new file mode 100644
index 0000000000000000000000000000000000000000..bd19bf52ed00282ae4799cbb4755b10d521c7aba
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da3aa36834dcde0763d096a1f44b3fb87909b0e22338d05200a86b7023ba19a2
+size 3147751
diff --git a/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc
new file mode 100644
index 0000000000000000000000000000000000000000..6f91be4953b568a5d2f25ae97d7204eae11ba92a
--- /dev/null
+++ b/moe-3b-qdp/param_split/encoder.layers.9.ffn.ffn.w_in.w.weight_input_compl_acc
@@ -0,0 +1 @@
+0.674227610146325
\ No newline at end of file