darabos commited on
Commit
39c4ace
·
1 Parent(s): 05891f5

Make a Position enum.

Browse files
lynxkite-app/src/lynxkite_app/crdt.py CHANGED
@@ -200,7 +200,7 @@ def try_to_load_workspace(ws: pycrdt.Map, name: str):
200
  name: Name of the workspace to load.
201
  """
202
  if os.path.exists(name):
203
- ws_pyd = workspace.load(name)
204
  crdt_update(
205
  ws,
206
  ws_pyd.model_dump(),
@@ -263,18 +263,18 @@ async def execute(name: str, ws_crdt: pycrdt.Map, ws_pyd: workspace.Workspace, d
263
  path = cwd / name
264
  assert path.is_relative_to(cwd), "Provided workspace path is invalid"
265
  # Save user changes before executing, in case the execution fails.
266
- workspace.save(ws_pyd, path)
267
  ops.load_user_scripts(name)
268
- workspace.connect_crdt(ws_pyd, ws_crdt)
269
- workspace.update_metadata(ws_pyd)
270
- if not workspace.has_executor(ws_pyd):
271
  return
272
  with ws_crdt.doc.transaction():
273
  for nc in ws_crdt["nodes"]:
274
  nc["data"]["status"] = "planned"
275
- ws_pyd = ws_pyd.normalize()
276
- await workspace.execute(ws_pyd)
277
- workspace.save(ws_pyd, path)
278
  print(f"Finished running {name} in {ws_pyd.env}.")
279
 
280
 
 
200
  name: Name of the workspace to load.
201
  """
202
  if os.path.exists(name):
203
+ ws_pyd = workspace.Workspace.load(name)
204
  crdt_update(
205
  ws,
206
  ws_pyd.model_dump(),
 
263
  path = cwd / name
264
  assert path.is_relative_to(cwd), "Provided workspace path is invalid"
265
  # Save user changes before executing, in case the execution fails.
266
+ ws_pyd.save(path)
267
  ops.load_user_scripts(name)
268
+ ws_pyd.connect_crdt(ws_crdt)
269
+ ws_pyd.update_metadata()
270
+ if not ws_pyd.has_executor():
271
  return
272
  with ws_crdt.doc.transaction():
273
  for nc in ws_crdt["nodes"]:
274
  nc["data"]["status"] = "planned"
275
+ ws_pyd.normalize()
276
+ await ws_pyd.execute()
277
+ ws_pyd.save(path)
278
  print(f"Finished running {name} in {ws_pyd.env}.")
279
 
280
 
lynxkite-app/src/lynxkite_app/main.py CHANGED
@@ -50,13 +50,13 @@ data_path = pathlib.Path()
50
  def save(req: SaveRequest):
51
  path = data_path / req.path
52
  assert path.is_relative_to(data_path)
53
- workspace.save(req.ws, path)
54
 
55
 
56
  @app.post("/api/save")
57
  async def save_and_execute(req: SaveRequest):
58
  save(req)
59
- await workspace.execute(req.ws)
60
  save(req)
61
  return req.ws
62
 
@@ -76,7 +76,7 @@ def load(path: str):
76
  assert path.is_relative_to(data_path)
77
  if not path.exists():
78
  return workspace.Workspace()
79
- return workspace.load(path)
80
 
81
 
82
  class DirectoryEntry(pydantic.BaseModel):
 
50
  def save(req: SaveRequest):
51
  path = data_path / req.path
52
  assert path.is_relative_to(data_path)
53
+ req.ws.save(path)
54
 
55
 
56
  @app.post("/api/save")
57
  async def save_and_execute(req: SaveRequest):
58
  save(req)
59
+ await req.ws.execute()
60
  save(req)
61
  return req.ws
62
 
 
76
  assert path.is_relative_to(data_path)
77
  if not path.exists():
78
  return workspace.Workspace()
79
+ return workspace.Workspace.load(path)
80
 
81
 
82
  class DirectoryEntry(pydantic.BaseModel):
lynxkite-core/src/lynxkite/core/ops.py CHANGED
@@ -95,17 +95,28 @@ class ParameterGroup(BaseConfig):
95
  type: str = "group"
96
 
97
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  class Input(BaseConfig):
99
  name: str
100
  type: Type
101
- # TODO: Make position an enum with the possible values.
102
- position: str = "left"
103
 
104
 
105
  class Output(BaseConfig):
106
  name: str
107
  type: Type
108
- position: str = "right"
109
 
110
 
111
  @dataclass
@@ -291,7 +302,7 @@ def input_position(**kwargs):
291
  def decorator(func):
292
  op = func.__op__
293
  for k, v in kwargs.items():
294
- op.get_input(k).position = v
295
  return func
296
 
297
  return decorator
@@ -303,7 +314,7 @@ def output_position(**kwargs):
303
  def decorator(func):
304
  op = func.__op__
305
  for k, v in kwargs.items():
306
- op.get_output(k).position = v
307
  return func
308
 
309
  return decorator
 
95
  type: str = "group"
96
 
97
 
98
+ class Position(enum.Enum):
99
+ """Defines the position of an input or output in the UI."""
100
+
101
+ LEFT = "left"
102
+ RIGHT = "right"
103
+ TOP = "top"
104
+ BOTTOM = "bottom"
105
+
106
+ def is_vertical(self):
107
+ return self in (self.TOP, self.BOTTOM)
108
+
109
+
110
  class Input(BaseConfig):
111
  name: str
112
  type: Type
113
+ position: Position = Position.LEFT
 
114
 
115
 
116
  class Output(BaseConfig):
117
  name: str
118
  type: Type
119
+ position: Position = Position.RIGHT
120
 
121
 
122
  @dataclass
 
302
  def decorator(func):
303
  op = func.__op__
304
  for k, v in kwargs.items():
305
+ op.get_input(k).position = Position(v)
306
  return func
307
 
308
  return decorator
 
314
  def decorator(func):
315
  op = func.__op__
316
  for k, v in kwargs.items():
317
+ op.get_output(k).position = Position(v)
318
  return func
319
 
320
  return decorator
lynxkite-core/src/lynxkite/core/workspace.py CHANGED
@@ -110,102 +110,91 @@ class Workspace(BaseConfig):
110
  valid_targets.add((n.id, h.name))
111
  for h in _ops[n.id].outputs:
112
  valid_sources.add((n.id, h.name))
113
- edges = [
114
  edge
115
  for edge in self.edges
116
  if (edge.source, edge.sourceHandle) in valid_sources
117
  and (edge.target, edge.targetHandle) in valid_targets
118
  ]
119
- return self.model_copy(update={"edges": edges})
120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
 
122
- def has_executor(ws: Workspace):
123
- return ws.env in ops.EXECUTORS
124
-
125
-
126
- async def execute(ws: Workspace):
127
- await ops.EXECUTORS[ws.env](ws)
128
-
129
-
130
- def save(ws: Workspace, path: str):
131
- """Persist a workspace to a local file in JSON format."""
132
- j = ws.model_dump()
133
- j = json.dumps(j, indent=2, sort_keys=True) + "\n"
134
- dirname, basename = os.path.split(path)
135
- if dirname:
136
- os.makedirs(dirname, exist_ok=True)
137
- # Create temp file in the same directory to make sure it's on the same filesystem.
138
- with tempfile.NamedTemporaryFile(
139
- "w", encoding="utf-8", prefix=f".{basename}.", dir=dirname, delete=False
140
- ) as f:
141
- temp_name = f.name
142
- f.write(j)
143
- os.replace(temp_name, path)
144
-
145
-
146
- def load(path: str) -> Workspace:
147
- """Load a workspace from a file.
148
-
149
- After loading the workspace, the metadata of the workspace is updated.
150
-
151
- Args:
152
- path (str): The path to the file to load the workspace from.
153
-
154
- Returns:
155
- Workspace: The loaded workspace object, with updated metadata.
156
- """
157
- with open(path, encoding="utf-8") as f:
158
- j = f.read()
159
- ws = Workspace.model_validate_json(j)
160
- # Metadata is added after loading. This way code changes take effect on old boxes too.
161
- update_metadata(ws)
162
- return ws
163
-
164
-
165
- def update_metadata(ws: Workspace) -> Workspace:
166
- """Update the metadata of the given workspace object.
167
-
168
- The metadata is the information about the operations that the nodes in the workspace represent,
169
- like the parameters and their possible values.
170
- This information comes from the catalog of operations for the environment of the workspace.
171
-
172
- Args:
173
- ws: The workspace object to update.
174
 
175
- Returns:
176
- Workspace: The updated workspace object.
177
- """
178
- if ws.env not in ops.CATALOGS:
179
- return ws
180
- catalog = ops.CATALOGS[ws.env]
181
- for node in ws.nodes:
182
- data = node.data
183
- op = catalog.get(data.title)
184
- if op:
185
- if data.meta != op:
186
- data.meta = op
187
- if hasattr(node, "_crdt"):
188
- node._crdt["data"]["meta"] = op.model_dump()
189
- if node.type != op.type:
190
- node.type = op.type
191
- if hasattr(node, "_crdt"):
192
- node._crdt["type"] = op.type
193
- if data.error == "Unknown operation.":
194
- data.error = None
 
 
 
 
 
 
 
195
  if hasattr(node, "_crdt"):
196
- node._crdt["data"]["error"] = None
197
- else:
198
- data.error = "Unknown operation."
199
- if hasattr(node, "_crdt"):
200
- node._crdt["data"]["meta"] = {}
201
- node._crdt["data"]["error"] = "Unknown operation."
202
- return ws
203
-
204
-
205
- def connect_crdt(ws_pyd: Workspace, ws_crdt: pycrdt.Map):
206
- ws_pyd._crdt = ws_crdt
207
- with ws_crdt.doc.transaction():
208
- for nc, np in zip(ws_crdt["nodes"], ws_pyd.nodes):
209
- if "data" not in nc:
210
- nc["data"] = pycrdt.Map()
211
- np._crdt = nc
 
110
  valid_targets.add((n.id, h.name))
111
  for h in _ops[n.id].outputs:
112
  valid_sources.add((n.id, h.name))
113
+ self.edges = [
114
  edge
115
  for edge in self.edges
116
  if (edge.source, edge.sourceHandle) in valid_sources
117
  and (edge.target, edge.targetHandle) in valid_targets
118
  ]
 
119
 
120
+ def has_executor(self):
121
+ return self.env in ops.EXECUTORS
122
+
123
+ async def execute(self):
124
+ await ops.EXECUTORS[self.env](self)
125
+
126
+ def save(self, path: str):
127
+ """Persist the workspace to a local file in JSON format."""
128
+ j = self.model_dump()
129
+ j = json.dumps(j, indent=2, sort_keys=True) + "\n"
130
+ dirname, basename = os.path.split(path)
131
+ if dirname:
132
+ os.makedirs(dirname, exist_ok=True)
133
+ # Create temp file in the same directory to make sure it's on the same filesystem.
134
+ with tempfile.NamedTemporaryFile(
135
+ "w", encoding="utf-8", prefix=f".{basename}.", dir=dirname, delete=False
136
+ ) as f:
137
+ temp_name = f.name
138
+ f.write(j)
139
+ os.replace(temp_name, path)
140
+
141
+ @staticmethod
142
+ def load(path: str) -> "Workspace":
143
+ """Load a workspace from a file.
144
+
145
+ After loading the workspace, the metadata of the workspace is updated.
146
+
147
+ Args:
148
+ path (str): The path to the file to load the workspace from.
149
+
150
+ Returns:
151
+ Workspace: The loaded workspace object, with updated metadata.
152
+ """
153
+ with open(path, encoding="utf-8") as f:
154
+ j = f.read()
155
+ ws = Workspace.model_validate_json(j)
156
+ # Metadata is added after loading. This way code changes take effect on old boxes too.
157
+ ws.update_metadata()
158
+ return ws
159
 
160
+ def update_metadata(self):
161
+ """Update the metadata of this workspace.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
 
163
+ The metadata is the information about the operations that the nodes in the workspace represent,
164
+ like the parameters and their possible values.
165
+ This information comes from the catalog of operations for the environment of the workspace.
166
+ """
167
+ if self.env not in ops.CATALOGS:
168
+ return self
169
+ catalog = ops.CATALOGS[self.env]
170
+ for node in self.nodes:
171
+ data = node.data
172
+ op = catalog.get(data.title)
173
+ if op:
174
+ if data.meta != op:
175
+ data.meta = op
176
+ if hasattr(node, "_crdt"):
177
+ # Go through JSON to simplify the types. CRDT can't handle enums.
178
+ node._crdt["data"]["meta"] = json.loads(op.model_dump_json())
179
+ print("set metadata to", op)
180
+ if node.type != op.type:
181
+ node.type = op.type
182
+ if hasattr(node, "_crdt"):
183
+ node._crdt["type"] = op.type
184
+ if data.error == "Unknown operation.":
185
+ data.error = None
186
+ if hasattr(node, "_crdt"):
187
+ node._crdt["data"]["error"] = None
188
+ else:
189
+ data.error = "Unknown operation."
190
  if hasattr(node, "_crdt"):
191
+ node._crdt["data"]["meta"] = {}
192
+ node._crdt["data"]["error"] = "Unknown operation."
193
+
194
+ def connect_crdt(self, ws_crdt: pycrdt.Map):
195
+ self._crdt = ws_crdt
196
+ with ws_crdt.doc.transaction():
197
+ for nc, np in zip(ws_crdt["nodes"], self.nodes):
198
+ if "data" not in nc:
199
+ nc["data"] = pycrdt.Map()
200
+ np._crdt = nc
 
 
 
 
 
 
lynxkite-core/tests/test_workspace.py CHANGED
@@ -34,9 +34,9 @@ def test_save_load():
34
  path = os.path.join(tempfile.gettempdir(), "test_workspace.json")
35
 
36
  try:
37
- workspace.save(ws, path)
38
  assert os.path.exists(path)
39
- loaded_ws = workspace.load(path)
40
  assert loaded_ws.env == ws.env
41
  assert len(loaded_ws.nodes) == len(ws.nodes)
42
  assert len(loaded_ws.edges) == len(ws.edges)
@@ -88,14 +88,14 @@ def test_update_metadata():
88
  position=workspace.Position(x=0, y=0),
89
  )
90
  )
91
- updated_ws = workspace._update_metadata(ws)
92
- assert updated_ws.nodes[0].data.meta.name == "Test Operation"
93
- assert updated_ws.nodes[0].data.error is None
94
- assert not hasattr(updated_ws.nodes[1].data, "meta")
95
- assert updated_ws.nodes[1].data.error == "Unknown operation."
96
 
97
 
98
  def test_update_metadata_with_empty_workspace():
99
  ws = workspace.Workspace(env="test")
100
- updated_ws = workspace._update_metadata(ws)
101
- assert len(updated_ws.nodes) == 0
 
34
  path = os.path.join(tempfile.gettempdir(), "test_workspace.json")
35
 
36
  try:
37
+ ws.save(path)
38
  assert os.path.exists(path)
39
+ loaded_ws = workspace.Workspace.load(path)
40
  assert loaded_ws.env == ws.env
41
  assert len(loaded_ws.nodes) == len(ws.nodes)
42
  assert len(loaded_ws.edges) == len(ws.edges)
 
88
  position=workspace.Position(x=0, y=0),
89
  )
90
  )
91
+ ws.update_metadata()
92
+ assert ws.nodes[0].data.meta.name == "Test Operation"
93
+ assert ws.nodes[0].data.error is None
94
+ assert not hasattr(ws.nodes[1].data, "meta")
95
+ assert ws.nodes[1].data.error == "Unknown operation."
96
 
97
 
98
  def test_update_metadata_with_empty_workspace():
99
  ws = workspace.Workspace(env="test")
100
+ ws.update_metadata()
101
+ assert len(ws.nodes) == 0
lynxkite-graph-analytics/src/lynxkite_graph_analytics/ml_ops.py CHANGED
@@ -22,7 +22,7 @@ def load_ws(model_workspace: str):
22
  path = cwd / model_workspace
23
  assert path.is_relative_to(cwd)
24
  assert path.exists(), f"Workspace {path} does not exist"
25
- ws = workspace.load(path)
26
  return ws
27
 
28
 
 
22
  path = cwd / model_workspace
23
  assert path.is_relative_to(cwd)
24
  assert path.exists(), f"Workspace {path} does not exist"
25
+ ws = workspace.Workspace.load(path)
26
  return ws
27
 
28
 
lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_core.py CHANGED
@@ -21,9 +21,9 @@ def op(name, weights=False, **kwargs):
21
  _op(func)
22
  op = func.__op__
23
  for p in op.inputs:
24
- p.position = "bottom"
25
  for p in op.outputs:
26
- p.position = "top"
27
  return func
28
 
29
  return decorator
 
21
  _op(func)
22
  op = func.__op__
23
  for p in op.inputs:
24
+ p.position = ops.Position.BOTTOM
25
  for p in op.outputs:
26
+ p.position = ops.Position.TOP
27
  return func
28
 
29
  return decorator
lynxkite-graph-analytics/src/lynxkite_graph_analytics/pytorch/pytorch_ops.py CHANGED
@@ -151,9 +151,9 @@ ops.register_passive_op(
151
  def _set_handle_positions(op):
152
  op: ops.Op = op.__op__
153
  for v in op.outputs:
154
- v.position = "top"
155
  for v in op.inputs:
156
- v.position = "bottom"
157
 
158
 
159
  def _register_simple_pytorch_layer(func):
 
151
  def _set_handle_positions(op):
152
  op: ops.Op = op.__op__
153
  for v in op.outputs:
154
+ v.position = ops.Position.TOP
155
  for v in op.inputs:
156
+ v.position = ops.Position.BOTTOM
157
 
158
 
159
  def _register_simple_pytorch_layer(func):
lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py CHANGED
@@ -875,7 +875,7 @@ async def get_chat_api(ws: str):
875
  path = cwd / ws
876
  assert path.is_relative_to(cwd)
877
  assert path.exists(), f"Workspace {path} does not exist"
878
- ws = workspace.load(path)
879
  contexts = await ops.EXECUTORS[ENV](ws)
880
  nodes = [op for op in ws.nodes if op.data.title == "LynxScribe RAG Graph Chatbot Backend"]
881
  [node] = nodes
@@ -939,7 +939,7 @@ def get_lynxscribe_workspaces():
939
  for p in pathlib.Path().glob("**/*"):
940
  if p.is_file():
941
  try:
942
- ws = workspace.load(p)
943
  if ws.env == ENV:
944
  workspaces.append(p)
945
  except Exception:
 
875
  path = cwd / ws
876
  assert path.is_relative_to(cwd)
877
  assert path.exists(), f"Workspace {path} does not exist"
878
+ ws = workspace.Workspace.load(path)
879
  contexts = await ops.EXECUTORS[ENV](ws)
880
  nodes = [op for op in ws.nodes if op.data.title == "LynxScribe RAG Graph Chatbot Backend"]
881
  [node] = nodes
 
939
  for p in pathlib.Path().glob("**/*"):
940
  if p.is_file():
941
  try:
942
+ ws = workspace.Workspace.load(p)
943
  if ws.env == ENV:
944
  workspaces.append(p)
945
  except Exception: