Upload splitters.py with huggingface_hub
Browse files- splitters.py +3 -3
splitters.py
CHANGED
|
@@ -36,7 +36,7 @@ class SplitRandomMix(Splitter):
|
|
| 36 |
def process(self, multi_stream: MultiStream) -> MultiStream:
|
| 37 |
mapping = {k: parse_random_mix_string(v) for k, v in self.mix.items()}
|
| 38 |
generators = random_mix_streams(multi_stream, mapping)
|
| 39 |
-
return MultiStream.from_generators(generators
|
| 40 |
|
| 41 |
|
| 42 |
class SeparateSplit(Splitter):
|
|
@@ -65,7 +65,7 @@ class SeparateSplit(Splitter):
|
|
| 65 |
if size:
|
| 66 |
so_far += size
|
| 67 |
generators = slice_streams(multi_stream, mapping)
|
| 68 |
-
return MultiStream.from_generators(generators
|
| 69 |
|
| 70 |
|
| 71 |
class SliceSplit(Splitter):
|
|
@@ -74,7 +74,7 @@ class SliceSplit(Splitter):
|
|
| 74 |
def process(self, multi_stream: MultiStream) -> MultiStream:
|
| 75 |
mapping = {k: parse_slices_string(v) for k, v in self.slices.items()}
|
| 76 |
generators = slice_streams(multi_stream, mapping)
|
| 77 |
-
return MultiStream.from_generators(generators
|
| 78 |
|
| 79 |
|
| 80 |
class Sampler(Artifact):
|
|
|
|
| 36 |
def process(self, multi_stream: MultiStream) -> MultiStream:
|
| 37 |
mapping = {k: parse_random_mix_string(v) for k, v in self.mix.items()}
|
| 38 |
generators = random_mix_streams(multi_stream, mapping)
|
| 39 |
+
return MultiStream.from_generators(generators)
|
| 40 |
|
| 41 |
|
| 42 |
class SeparateSplit(Splitter):
|
|
|
|
| 65 |
if size:
|
| 66 |
so_far += size
|
| 67 |
generators = slice_streams(multi_stream, mapping)
|
| 68 |
+
return MultiStream.from_generators(generators)
|
| 69 |
|
| 70 |
|
| 71 |
class SliceSplit(Splitter):
|
|
|
|
| 74 |
def process(self, multi_stream: MultiStream) -> MultiStream:
|
| 75 |
mapping = {k: parse_slices_string(v) for k, v in self.slices.items()}
|
| 76 |
generators = slice_streams(multi_stream, mapping)
|
| 77 |
+
return MultiStream.from_generators(generators)
|
| 78 |
|
| 79 |
|
| 80 |
class Sampler(Artifact):
|