Spaces:
Running
Running
Make a Position enum.
Browse files- lynxkite-app/src/lynxkite_app/crdt.py +8 -8
- lynxkite-app/src/lynxkite_app/main.py +3 -3
- lynxkite-core/src/lynxkite/core/ops.py +16 -5
- lynxkite-core/src/lynxkite/core/workspace.py +79 -90
- lynxkite-core/tests/test_workspace.py +9 -9
- lynxkite-graph-analytics/src/lynxkite_graph_analytics/ml_ops.py +1 -1
- lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_core.py +2 -2
- lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_ops.py +2 -2
- lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py +2 -2
lynxkite-app/src/lynxkite_app/crdt.py
CHANGED
@@ -200,7 +200,7 @@ def try_to_load_workspace(ws: pycrdt.Map, name: str):
|
|
200 |
name: Name of the workspace to load.
|
201 |
"""
|
202 |
if os.path.exists(name):
|
203 |
-
ws_pyd = workspace.load(name)
|
204 |
crdt_update(
|
205 |
ws,
|
206 |
ws_pyd.model_dump(),
|
@@ -263,18 +263,18 @@ async def execute(name: str, ws_crdt: pycrdt.Map, ws_pyd: workspace.Workspace, d
|
|
263 |
path = cwd / name
|
264 |
assert path.is_relative_to(cwd), "Provided workspace path is invalid"
|
265 |
# Save user changes before executing, in case the execution fails.
|
266 |
-
|
267 |
ops.load_user_scripts(name)
|
268 |
-
|
269 |
-
|
270 |
-
if not
|
271 |
return
|
272 |
with ws_crdt.doc.transaction():
|
273 |
for nc in ws_crdt["nodes"]:
|
274 |
nc["data"]["status"] = "planned"
|
275 |
-
ws_pyd
|
276 |
-
await
|
277 |
-
|
278 |
print(f"Finished running {name} in {ws_pyd.env}.")
|
279 |
|
280 |
|
|
|
200 |
name: Name of the workspace to load.
|
201 |
"""
|
202 |
if os.path.exists(name):
|
203 |
+
ws_pyd = workspace.Workspace.load(name)
|
204 |
crdt_update(
|
205 |
ws,
|
206 |
ws_pyd.model_dump(),
|
|
|
263 |
path = cwd / name
|
264 |
assert path.is_relative_to(cwd), "Provided workspace path is invalid"
|
265 |
# Save user changes before executing, in case the execution fails.
|
266 |
+
ws_pyd.save(path)
|
267 |
ops.load_user_scripts(name)
|
268 |
+
ws_pyd.connect_crdt(ws_crdt)
|
269 |
+
ws_pyd.update_metadata()
|
270 |
+
if not ws_pyd.has_executor():
|
271 |
return
|
272 |
with ws_crdt.doc.transaction():
|
273 |
for nc in ws_crdt["nodes"]:
|
274 |
nc["data"]["status"] = "planned"
|
275 |
+
ws_pyd.normalize()
|
276 |
+
await ws_pyd.execute()
|
277 |
+
ws_pyd.save(path)
|
278 |
print(f"Finished running {name} in {ws_pyd.env}.")
|
279 |
|
280 |
|
lynxkite-app/src/lynxkite_app/main.py
CHANGED
@@ -50,13 +50,13 @@ data_path = pathlib.Path()
|
|
50 |
def save(req: SaveRequest):
|
51 |
path = data_path / req.path
|
52 |
assert path.is_relative_to(data_path)
|
53 |
-
|
54 |
|
55 |
|
56 |
@app.post("/api/save")
|
57 |
async def save_and_execute(req: SaveRequest):
|
58 |
save(req)
|
59 |
-
await
|
60 |
save(req)
|
61 |
return req.ws
|
62 |
|
@@ -76,7 +76,7 @@ def load(path: str):
|
|
76 |
assert path.is_relative_to(data_path)
|
77 |
if not path.exists():
|
78 |
return workspace.Workspace()
|
79 |
-
return workspace.load(path)
|
80 |
|
81 |
|
82 |
class DirectoryEntry(pydantic.BaseModel):
|
|
|
50 |
def save(req: SaveRequest):
|
51 |
path = data_path / req.path
|
52 |
assert path.is_relative_to(data_path)
|
53 |
+
req.ws.save(path)
|
54 |
|
55 |
|
56 |
@app.post("/api/save")
|
57 |
async def save_and_execute(req: SaveRequest):
|
58 |
save(req)
|
59 |
+
await req.ws.execute()
|
60 |
save(req)
|
61 |
return req.ws
|
62 |
|
|
|
76 |
assert path.is_relative_to(data_path)
|
77 |
if not path.exists():
|
78 |
return workspace.Workspace()
|
79 |
+
return workspace.Workspace.load(path)
|
80 |
|
81 |
|
82 |
class DirectoryEntry(pydantic.BaseModel):
|
lynxkite-core/src/lynxkite/core/ops.py
CHANGED
@@ -95,17 +95,28 @@ class ParameterGroup(BaseConfig):
|
|
95 |
type: str = "group"
|
96 |
|
97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
class Input(BaseConfig):
|
99 |
name: str
|
100 |
type: Type
|
101 |
-
|
102 |
-
position: str = "left"
|
103 |
|
104 |
|
105 |
class Output(BaseConfig):
|
106 |
name: str
|
107 |
type: Type
|
108 |
-
position:
|
109 |
|
110 |
|
111 |
@dataclass
|
@@ -291,7 +302,7 @@ def input_position(**kwargs):
|
|
291 |
def decorator(func):
|
292 |
op = func.__op__
|
293 |
for k, v in kwargs.items():
|
294 |
-
op.get_input(k).position = v
|
295 |
return func
|
296 |
|
297 |
return decorator
|
@@ -303,7 +314,7 @@ def output_position(**kwargs):
|
|
303 |
def decorator(func):
|
304 |
op = func.__op__
|
305 |
for k, v in kwargs.items():
|
306 |
-
op.get_output(k).position = v
|
307 |
return func
|
308 |
|
309 |
return decorator
|
|
|
95 |
type: str = "group"
|
96 |
|
97 |
|
98 |
+
class Position(enum.Enum):
|
99 |
+
"""Defines the position of an input or output in the UI."""
|
100 |
+
|
101 |
+
LEFT = "left"
|
102 |
+
RIGHT = "right"
|
103 |
+
TOP = "top"
|
104 |
+
BOTTOM = "bottom"
|
105 |
+
|
106 |
+
def is_vertical(self):
|
107 |
+
return self in (self.TOP, self.BOTTOM)
|
108 |
+
|
109 |
+
|
110 |
class Input(BaseConfig):
|
111 |
name: str
|
112 |
type: Type
|
113 |
+
position: Position = Position.LEFT
|
|
|
114 |
|
115 |
|
116 |
class Output(BaseConfig):
|
117 |
name: str
|
118 |
type: Type
|
119 |
+
position: Position = Position.RIGHT
|
120 |
|
121 |
|
122 |
@dataclass
|
|
|
302 |
def decorator(func):
|
303 |
op = func.__op__
|
304 |
for k, v in kwargs.items():
|
305 |
+
op.get_input(k).position = Position(v)
|
306 |
return func
|
307 |
|
308 |
return decorator
|
|
|
314 |
def decorator(func):
|
315 |
op = func.__op__
|
316 |
for k, v in kwargs.items():
|
317 |
+
op.get_output(k).position = Position(v)
|
318 |
return func
|
319 |
|
320 |
return decorator
|
lynxkite-core/src/lynxkite/core/workspace.py
CHANGED
@@ -110,102 +110,91 @@ class Workspace(BaseConfig):
|
|
110 |
valid_targets.add((n.id, h.name))
|
111 |
for h in _ops[n.id].outputs:
|
112 |
valid_sources.add((n.id, h.name))
|
113 |
-
edges = [
|
114 |
edge
|
115 |
for edge in self.edges
|
116 |
if (edge.source, edge.sourceHandle) in valid_sources
|
117 |
and (edge.target, edge.targetHandle) in valid_targets
|
118 |
]
|
119 |
-
return self.model_copy(update={"edges": edges})
|
120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
121 |
|
122 |
-
def
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
async def execute(ws: Workspace):
|
127 |
-
await ops.EXECUTORS[ws.env](ws)
|
128 |
-
|
129 |
-
|
130 |
-
def save(ws: Workspace, path: str):
|
131 |
-
"""Persist a workspace to a local file in JSON format."""
|
132 |
-
j = ws.model_dump()
|
133 |
-
j = json.dumps(j, indent=2, sort_keys=True) + "\n"
|
134 |
-
dirname, basename = os.path.split(path)
|
135 |
-
if dirname:
|
136 |
-
os.makedirs(dirname, exist_ok=True)
|
137 |
-
# Create temp file in the same directory to make sure it's on the same filesystem.
|
138 |
-
with tempfile.NamedTemporaryFile(
|
139 |
-
"w", encoding="utf-8", prefix=f".{basename}.", dir=dirname, delete=False
|
140 |
-
) as f:
|
141 |
-
temp_name = f.name
|
142 |
-
f.write(j)
|
143 |
-
os.replace(temp_name, path)
|
144 |
-
|
145 |
-
|
146 |
-
def load(path: str) -> Workspace:
|
147 |
-
"""Load a workspace from a file.
|
148 |
-
|
149 |
-
After loading the workspace, the metadata of the workspace is updated.
|
150 |
-
|
151 |
-
Args:
|
152 |
-
path (str): The path to the file to load the workspace from.
|
153 |
-
|
154 |
-
Returns:
|
155 |
-
Workspace: The loaded workspace object, with updated metadata.
|
156 |
-
"""
|
157 |
-
with open(path, encoding="utf-8") as f:
|
158 |
-
j = f.read()
|
159 |
-
ws = Workspace.model_validate_json(j)
|
160 |
-
# Metadata is added after loading. This way code changes take effect on old boxes too.
|
161 |
-
update_metadata(ws)
|
162 |
-
return ws
|
163 |
-
|
164 |
-
|
165 |
-
def update_metadata(ws: Workspace) -> Workspace:
|
166 |
-
"""Update the metadata of the given workspace object.
|
167 |
-
|
168 |
-
The metadata is the information about the operations that the nodes in the workspace represent,
|
169 |
-
like the parameters and their possible values.
|
170 |
-
This information comes from the catalog of operations for the environment of the workspace.
|
171 |
-
|
172 |
-
Args:
|
173 |
-
ws: The workspace object to update.
|
174 |
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
if
|
186 |
-
data.meta
|
187 |
-
|
188 |
-
node
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
195 |
if hasattr(node, "_crdt"):
|
196 |
-
node._crdt["data"]["
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
ws_pyd._crdt = ws_crdt
|
207 |
-
with ws_crdt.doc.transaction():
|
208 |
-
for nc, np in zip(ws_crdt["nodes"], ws_pyd.nodes):
|
209 |
-
if "data" not in nc:
|
210 |
-
nc["data"] = pycrdt.Map()
|
211 |
-
np._crdt = nc
|
|
|
110 |
valid_targets.add((n.id, h.name))
|
111 |
for h in _ops[n.id].outputs:
|
112 |
valid_sources.add((n.id, h.name))
|
113 |
+
self.edges = [
|
114 |
edge
|
115 |
for edge in self.edges
|
116 |
if (edge.source, edge.sourceHandle) in valid_sources
|
117 |
and (edge.target, edge.targetHandle) in valid_targets
|
118 |
]
|
|
|
119 |
|
120 |
+
def has_executor(self):
|
121 |
+
return self.env in ops.EXECUTORS
|
122 |
+
|
123 |
+
async def execute(self):
|
124 |
+
await ops.EXECUTORS[self.env](self)
|
125 |
+
|
126 |
+
def save(self, path: str):
|
127 |
+
"""Persist the workspace to a local file in JSON format."""
|
128 |
+
j = self.model_dump()
|
129 |
+
j = json.dumps(j, indent=2, sort_keys=True) + "\n"
|
130 |
+
dirname, basename = os.path.split(path)
|
131 |
+
if dirname:
|
132 |
+
os.makedirs(dirname, exist_ok=True)
|
133 |
+
# Create temp file in the same directory to make sure it's on the same filesystem.
|
134 |
+
with tempfile.NamedTemporaryFile(
|
135 |
+
"w", encoding="utf-8", prefix=f".{basename}.", dir=dirname, delete=False
|
136 |
+
) as f:
|
137 |
+
temp_name = f.name
|
138 |
+
f.write(j)
|
139 |
+
os.replace(temp_name, path)
|
140 |
+
|
141 |
+
@staticmethod
|
142 |
+
def load(path: str) -> "Workspace":
|
143 |
+
"""Load a workspace from a file.
|
144 |
+
|
145 |
+
After loading the workspace, the metadata of the workspace is updated.
|
146 |
+
|
147 |
+
Args:
|
148 |
+
path (str): The path to the file to load the workspace from.
|
149 |
+
|
150 |
+
Returns:
|
151 |
+
Workspace: The loaded workspace object, with updated metadata.
|
152 |
+
"""
|
153 |
+
with open(path, encoding="utf-8") as f:
|
154 |
+
j = f.read()
|
155 |
+
ws = Workspace.model_validate_json(j)
|
156 |
+
# Metadata is added after loading. This way code changes take effect on old boxes too.
|
157 |
+
ws.update_metadata()
|
158 |
+
return ws
|
159 |
|
160 |
+
def update_metadata(self):
|
161 |
+
"""Update the metadata of this workspace.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
162 |
|
163 |
+
The metadata is the information about the operations that the nodes in the workspace represent,
|
164 |
+
like the parameters and their possible values.
|
165 |
+
This information comes from the catalog of operations for the environment of the workspace.
|
166 |
+
"""
|
167 |
+
if self.env not in ops.CATALOGS:
|
168 |
+
return self
|
169 |
+
catalog = ops.CATALOGS[self.env]
|
170 |
+
for node in self.nodes:
|
171 |
+
data = node.data
|
172 |
+
op = catalog.get(data.title)
|
173 |
+
if op:
|
174 |
+
if data.meta != op:
|
175 |
+
data.meta = op
|
176 |
+
if hasattr(node, "_crdt"):
|
177 |
+
# Go through JSON to simplify the types. CRDT can't handle enums.
|
178 |
+
node._crdt["data"]["meta"] = json.loads(op.model_dump_json())
|
179 |
+
print("set metadata to", op)
|
180 |
+
if node.type != op.type:
|
181 |
+
node.type = op.type
|
182 |
+
if hasattr(node, "_crdt"):
|
183 |
+
node._crdt["type"] = op.type
|
184 |
+
if data.error == "Unknown operation.":
|
185 |
+
data.error = None
|
186 |
+
if hasattr(node, "_crdt"):
|
187 |
+
node._crdt["data"]["error"] = None
|
188 |
+
else:
|
189 |
+
data.error = "Unknown operation."
|
190 |
if hasattr(node, "_crdt"):
|
191 |
+
node._crdt["data"]["meta"] = {}
|
192 |
+
node._crdt["data"]["error"] = "Unknown operation."
|
193 |
+
|
194 |
+
def connect_crdt(self, ws_crdt: pycrdt.Map):
|
195 |
+
self._crdt = ws_crdt
|
196 |
+
with ws_crdt.doc.transaction():
|
197 |
+
for nc, np in zip(ws_crdt["nodes"], self.nodes):
|
198 |
+
if "data" not in nc:
|
199 |
+
nc["data"] = pycrdt.Map()
|
200 |
+
np._crdt = nc
|
|
|
|
|
|
|
|
|
|
|
|
lynxkite-core/tests/test_workspace.py
CHANGED
@@ -34,9 +34,9 @@ def test_save_load():
|
|
34 |
path = os.path.join(tempfile.gettempdir(), "test_workspace.json")
|
35 |
|
36 |
try:
|
37 |
-
|
38 |
assert os.path.exists(path)
|
39 |
-
loaded_ws = workspace.load(path)
|
40 |
assert loaded_ws.env == ws.env
|
41 |
assert len(loaded_ws.nodes) == len(ws.nodes)
|
42 |
assert len(loaded_ws.edges) == len(ws.edges)
|
@@ -88,14 +88,14 @@ def test_update_metadata():
|
|
88 |
position=workspace.Position(x=0, y=0),
|
89 |
)
|
90 |
)
|
91 |
-
|
92 |
-
assert
|
93 |
-
assert
|
94 |
-
assert not hasattr(
|
95 |
-
assert
|
96 |
|
97 |
|
98 |
def test_update_metadata_with_empty_workspace():
|
99 |
ws = workspace.Workspace(env="test")
|
100 |
-
|
101 |
-
assert len(
|
|
|
34 |
path = os.path.join(tempfile.gettempdir(), "test_workspace.json")
|
35 |
|
36 |
try:
|
37 |
+
ws.save(path)
|
38 |
assert os.path.exists(path)
|
39 |
+
loaded_ws = workspace.Workspace.load(path)
|
40 |
assert loaded_ws.env == ws.env
|
41 |
assert len(loaded_ws.nodes) == len(ws.nodes)
|
42 |
assert len(loaded_ws.edges) == len(ws.edges)
|
|
|
88 |
position=workspace.Position(x=0, y=0),
|
89 |
)
|
90 |
)
|
91 |
+
ws.update_metadata()
|
92 |
+
assert ws.nodes[0].data.meta.name == "Test Operation"
|
93 |
+
assert ws.nodes[0].data.error is None
|
94 |
+
assert not hasattr(ws.nodes[1].data, "meta")
|
95 |
+
assert ws.nodes[1].data.error == "Unknown operation."
|
96 |
|
97 |
|
98 |
def test_update_metadata_with_empty_workspace():
|
99 |
ws = workspace.Workspace(env="test")
|
100 |
+
ws.update_metadata()
|
101 |
+
assert len(ws.nodes) == 0
|
lynxkite-graph-analytics/src/lynxkite_graph_analytics/ml_ops.py
CHANGED
@@ -22,7 +22,7 @@ def load_ws(model_workspace: str):
|
|
22 |
path = cwd / model_workspace
|
23 |
assert path.is_relative_to(cwd)
|
24 |
assert path.exists(), f"Workspace {path} does not exist"
|
25 |
-
ws = workspace.load(path)
|
26 |
return ws
|
27 |
|
28 |
|
|
|
22 |
path = cwd / model_workspace
|
23 |
assert path.is_relative_to(cwd)
|
24 |
assert path.exists(), f"Workspace {path} does not exist"
|
25 |
+
ws = workspace.Workspace.load(path)
|
26 |
return ws
|
27 |
|
28 |
|
lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_core.py
CHANGED
@@ -21,9 +21,9 @@ def op(name, weights=False, **kwargs):
|
|
21 |
_op(func)
|
22 |
op = func.__op__
|
23 |
for p in op.inputs:
|
24 |
-
p.position =
|
25 |
for p in op.outputs:
|
26 |
-
p.position =
|
27 |
return func
|
28 |
|
29 |
return decorator
|
|
|
21 |
_op(func)
|
22 |
op = func.__op__
|
23 |
for p in op.inputs:
|
24 |
+
p.position = ops.Position.BOTTOM
|
25 |
for p in op.outputs:
|
26 |
+
p.position = ops.Position.TOP
|
27 |
return func
|
28 |
|
29 |
return decorator
|
lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_ops.py
CHANGED
@@ -151,9 +151,9 @@ ops.register_passive_op(
|
|
151 |
def _set_handle_positions(op):
|
152 |
op: ops.Op = op.__op__
|
153 |
for v in op.outputs:
|
154 |
-
v.position =
|
155 |
for v in op.inputs:
|
156 |
-
v.position =
|
157 |
|
158 |
|
159 |
def _register_simple_pytorch_layer(func):
|
|
|
151 |
def _set_handle_positions(op):
|
152 |
op: ops.Op = op.__op__
|
153 |
for v in op.outputs:
|
154 |
+
v.position = ops.Position.TOP
|
155 |
for v in op.inputs:
|
156 |
+
v.position = ops.Position.BOTTOM
|
157 |
|
158 |
|
159 |
def _register_simple_pytorch_layer(func):
|
lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py
CHANGED
@@ -875,7 +875,7 @@ async def get_chat_api(ws: str):
|
|
875 |
path = cwd / ws
|
876 |
assert path.is_relative_to(cwd)
|
877 |
assert path.exists(), f"Workspace {path} does not exist"
|
878 |
-
ws = workspace.load(path)
|
879 |
contexts = await ops.EXECUTORS[ENV](ws)
|
880 |
nodes = [op for op in ws.nodes if op.data.title == "LynxScribe RAG Graph Chatbot Backend"]
|
881 |
[node] = nodes
|
@@ -939,7 +939,7 @@ def get_lynxscribe_workspaces():
|
|
939 |
for p in pathlib.Path().glob("**/*"):
|
940 |
if p.is_file():
|
941 |
try:
|
942 |
-
ws = workspace.load(p)
|
943 |
if ws.env == ENV:
|
944 |
workspaces.append(p)
|
945 |
except Exception:
|
|
|
875 |
path = cwd / ws
|
876 |
assert path.is_relative_to(cwd)
|
877 |
assert path.exists(), f"Workspace {path} does not exist"
|
878 |
+
ws = workspace.Workspace.load(path)
|
879 |
contexts = await ops.EXECUTORS[ENV](ws)
|
880 |
nodes = [op for op in ws.nodes if op.data.title == "LynxScribe RAG Graph Chatbot Backend"]
|
881 |
[node] = nodes
|
|
|
939 |
for p in pathlib.Path().glob("**/*"):
|
940 |
if p.is_file():
|
941 |
try:
|
942 |
+
ws = workspace.Workspace.load(p)
|
943 |
if ws.env == ENV:
|
944 |
workspaces.append(p)
|
945 |
except Exception:
|