a whole ass Thing over here
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
from .catalog import CombinedCatalog
|
||||
from .connections import ConnectionRegistry, parse_connection_id, qualify_node_name
|
||||
from .models import (
|
||||
AuthRecord,
|
||||
CatalogNodeEntry,
|
||||
CatalogSnapshot,
|
||||
ConnectionConfig,
|
||||
RawWorkflowPlan,
|
||||
)
|
||||
from .service import WfMcpService
|
||||
from .store import FileStore, Store
|
||||
|
||||
__all__ = [
|
||||
"AuthRecord",
|
||||
"CatalogNodeEntry",
|
||||
"CatalogSnapshot",
|
||||
"CombinedCatalog",
|
||||
"ConnectionConfig",
|
||||
"ConnectionRegistry",
|
||||
"FileStore",
|
||||
"RawWorkflowPlan",
|
||||
"Store",
|
||||
"WfMcpService",
|
||||
"parse_connection_id",
|
||||
"qualify_node_name",
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from wf_authoring import NodeCatalog, NodeSpec
|
||||
|
||||
from .connections import qualify_node_name
|
||||
from .models import CatalogNodeEntry, CatalogSnapshot
|
||||
|
||||
|
||||
def snapshot_from_specs(
|
||||
connection_id: str,
|
||||
*,
|
||||
specs: dict[str, NodeSpec[Any, Any]],
|
||||
fetched_at_epoch_ms: int,
|
||||
max_age_seconds: int,
|
||||
) -> CatalogSnapshot:
|
||||
catalog = NodeCatalog.from_specs(*specs.values())
|
||||
nodes = [
|
||||
CatalogNodeEntry(
|
||||
qualified_name=entry.name
|
||||
if entry.name.startswith(f"{connection_id}.")
|
||||
else qualify_node_name(connection_id, entry.name),
|
||||
connection_id=connection_id,
|
||||
local_name=entry.name.removeprefix(f"{connection_id}."),
|
||||
description=entry.description,
|
||||
outcomes=entry.outcomes,
|
||||
input_schema=entry.input_schema,
|
||||
output_schema=entry.output_schema,
|
||||
)
|
||||
for entry in catalog.entries()
|
||||
]
|
||||
return CatalogSnapshot(
|
||||
connection_id=connection_id,
|
||||
fetched_at_epoch_ms=fetched_at_epoch_ms,
|
||||
max_age_seconds=max_age_seconds,
|
||||
nodes=nodes,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CombinedCatalog:
|
||||
snapshots: dict[str, CatalogSnapshot] = field(default_factory=dict)
|
||||
|
||||
def entries(self) -> list[CatalogNodeEntry]:
|
||||
result: list[CatalogNodeEntry] = []
|
||||
for snapshot in self.snapshots.values():
|
||||
result.extend(snapshot.nodes)
|
||||
return sorted(result, key=lambda entry: entry.qualified_name)
|
||||
|
||||
def as_payload(self) -> dict[str, Any]:
|
||||
return {
|
||||
"nodes": [
|
||||
{
|
||||
"qualified_name": entry.qualified_name,
|
||||
"connection_id": entry.connection_id,
|
||||
"local_name": entry.local_name,
|
||||
"description": entry.description,
|
||||
"outcomes": list(entry.outcomes),
|
||||
"input_schema": entry.input_schema,
|
||||
"output_schema": entry.output_schema,
|
||||
}
|
||||
for entry in self.entries()
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .models import ConnectionConfig
|
||||
|
||||
|
||||
def parse_connection_id(connection_id: str) -> tuple[str, str]:
|
||||
if "." not in connection_id:
|
||||
raise ValueError(
|
||||
"connection id must look like '<server>.<account>'"
|
||||
)
|
||||
server, account = connection_id.split(".", 1)
|
||||
if not server or not account:
|
||||
raise ValueError(
|
||||
"connection id must look like '<server>.<account>'"
|
||||
)
|
||||
return server, account
|
||||
|
||||
|
||||
def qualify_node_name(connection_id: str, local_name: str) -> str:
|
||||
parse_connection_id(connection_id)
|
||||
if not local_name:
|
||||
raise ValueError("local node name must not be empty")
|
||||
return f"{connection_id}.{local_name}"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ConnectionRegistry:
|
||||
connections: dict[str, ConnectionConfig] = field(default_factory=dict)
|
||||
|
||||
def register(self, connection: ConnectionConfig) -> None:
|
||||
parse_connection_id(connection.id)
|
||||
self.connections[connection.id] = connection
|
||||
|
||||
def get(self, connection_id: str) -> ConnectionConfig:
|
||||
return self.connections[connection_id]
|
||||
|
||||
def list_enabled(self) -> list[ConnectionConfig]:
|
||||
return [connection for connection in self.connections.values() if connection.enabled]
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ConnectionConfig:
|
||||
id: str
|
||||
server: str
|
||||
account: str
|
||||
enabled: bool = True
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AuthRecord:
|
||||
connection_id: str
|
||||
scheme: str
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CatalogNodeEntry:
|
||||
qualified_name: str
|
||||
connection_id: str
|
||||
local_name: str
|
||||
description: str | None
|
||||
outcomes: tuple[str, ...]
|
||||
input_schema: dict[str, Any]
|
||||
output_schema: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CatalogSnapshot:
|
||||
connection_id: str
|
||||
fetched_at_epoch_ms: int
|
||||
max_age_seconds: int
|
||||
nodes: list[CatalogNodeEntry] = field(default_factory=list)
|
||||
|
||||
def is_stale(self, now_epoch_ms: int) -> bool:
|
||||
age_ms = now_epoch_ms - self.fetched_at_epoch_ms
|
||||
return age_ms > self.max_age_seconds * 1000
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RawWorkflowPlan:
|
||||
name: str
|
||||
input_schema: dict[str, Any]
|
||||
state_schema: dict[str, Any]
|
||||
output_schema: dict[str, Any]
|
||||
start: str
|
||||
nodes: list[dict[str, Any]]
|
||||
edges: list[dict[str, Any]]
|
||||
|
||||
|
||||
def dump_catalog_snapshot(snapshot: CatalogSnapshot) -> dict[str, Any]:
|
||||
return {
|
||||
"connection_id": snapshot.connection_id,
|
||||
"fetched_at_epoch_ms": snapshot.fetched_at_epoch_ms,
|
||||
"max_age_seconds": snapshot.max_age_seconds,
|
||||
"nodes": [asdict(node) for node in snapshot.nodes],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from wf_authoring import NodeSpec, build_async_registry
|
||||
from wf_core import NodeUse, Workflow, execute_workflow_async
|
||||
|
||||
from .catalog import CombinedCatalog, snapshot_from_specs
|
||||
from .connections import ConnectionRegistry, parse_connection_id, qualify_node_name
|
||||
from .models import AuthRecord, CatalogSnapshot, ConnectionConfig, RawWorkflowPlan
|
||||
from .store import Store
|
||||
|
||||
|
||||
def _qualify_spec(connection_id: str, spec: NodeSpec[Any, Any]) -> NodeSpec[Any, Any]:
|
||||
return NodeSpec(
|
||||
name=qualify_node_name(connection_id, spec.name),
|
||||
input_model=spec.input_model,
|
||||
output_model=spec.output_model,
|
||||
outcomes=spec.outcomes,
|
||||
fn=spec.fn,
|
||||
description=spec.description,
|
||||
is_async=spec.is_async,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class WfMcpService:
|
||||
store: Store
|
||||
default_catalog_max_age_seconds: int = 300
|
||||
connections: ConnectionRegistry = field(default_factory=ConnectionRegistry)
|
||||
specs_by_connection: dict[str, dict[str, NodeSpec[Any, Any]]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
def register_connection(self, connection: ConnectionConfig) -> None:
|
||||
parse_connection_id(connection.id)
|
||||
self.connections.register(connection)
|
||||
|
||||
def save_auth(self, record: AuthRecord) -> None:
|
||||
self.store.save_auth(record)
|
||||
|
||||
def load_auth(self, connection_id: str) -> AuthRecord | None:
|
||||
return self.store.load_auth(connection_id)
|
||||
|
||||
def register_specs(
|
||||
self,
|
||||
connection_id: str,
|
||||
*specs: NodeSpec[Any, Any],
|
||||
max_age_seconds: int | None = None,
|
||||
) -> None:
|
||||
self.connections.get(connection_id)
|
||||
qualified_specs = {
|
||||
qualify_node_name(connection_id, spec.name): _qualify_spec(connection_id, spec)
|
||||
for spec in specs
|
||||
}
|
||||
self.specs_by_connection[connection_id] = qualified_specs
|
||||
snapshot = snapshot_from_specs(
|
||||
connection_id,
|
||||
specs=qualified_specs,
|
||||
fetched_at_epoch_ms=int(time.time() * 1000),
|
||||
max_age_seconds=max_age_seconds or self.default_catalog_max_age_seconds,
|
||||
)
|
||||
self.store.save_catalog(snapshot)
|
||||
|
||||
def get_catalog(self) -> CombinedCatalog:
|
||||
snapshots: dict[str, CatalogSnapshot] = {}
|
||||
for connection in self.connections.list_enabled():
|
||||
snapshot = self.store.load_catalog(connection.id)
|
||||
if snapshot is not None:
|
||||
snapshots[connection.id] = snapshot
|
||||
return CombinedCatalog(snapshots=snapshots)
|
||||
|
||||
def compile_plan(self, plan: RawWorkflowPlan) -> Workflow:
|
||||
node_defs: dict[str, Any] = {}
|
||||
for step in plan.nodes:
|
||||
if step.get("type") != "node":
|
||||
continue
|
||||
qualified_name = step["node"]
|
||||
spec = self._get_qualified_spec(qualified_name)
|
||||
node_defs[qualified_name] = spec.to_node_def()
|
||||
|
||||
payload = {
|
||||
"name": plan.name,
|
||||
"input_schema": plan.input_schema,
|
||||
"state_schema": plan.state_schema,
|
||||
"output_schema": plan.output_schema,
|
||||
"start": plan.start,
|
||||
"node_defs": [node.model_dump() for node in node_defs.values()],
|
||||
"nodes": plan.nodes,
|
||||
"edges": plan.edges,
|
||||
}
|
||||
return Workflow.model_validate(payload)
|
||||
|
||||
async def run_workflow_from_plan(
|
||||
self,
|
||||
plan: RawWorkflowPlan,
|
||||
workflow_input: dict[str, Any],
|
||||
):
|
||||
workflow = self.compile_plan(plan)
|
||||
specs = [
|
||||
self._get_qualified_spec(node.node)
|
||||
for node in workflow.nodes
|
||||
if isinstance(node, NodeUse)
|
||||
]
|
||||
registry = build_async_registry(*specs)
|
||||
return await execute_workflow_async(workflow, workflow_input, registry)
|
||||
|
||||
def _get_qualified_spec(self, qualified_name: str) -> NodeSpec[Any, Any]:
|
||||
connection_id, _ = qualified_name.rsplit(".", 1)
|
||||
specs = self.specs_by_connection.get(connection_id)
|
||||
if specs is None or qualified_name not in specs:
|
||||
raise KeyError(f"unknown qualified node {qualified_name!r}")
|
||||
return specs[qualified_name]
|
||||
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from .models import AuthRecord, CatalogNodeEntry, CatalogSnapshot
|
||||
|
||||
|
||||
class Store:
|
||||
def save_auth(self, record: AuthRecord) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def load_auth(self, connection_id: str) -> AuthRecord | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def save_catalog(self, snapshot: CatalogSnapshot) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def load_catalog(self, connection_id: str) -> CatalogSnapshot | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileStore(Store):
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self.auth_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.catalog_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@property
|
||||
def auth_dir(self) -> Path:
|
||||
return self.root / "auth"
|
||||
|
||||
@property
|
||||
def catalog_dir(self) -> Path:
|
||||
return self.root / "catalog"
|
||||
|
||||
def _auth_path(self, connection_id: str) -> Path:
|
||||
return self.auth_dir / f"{connection_id}.json"
|
||||
|
||||
def _catalog_path(self, connection_id: str) -> Path:
|
||||
return self.catalog_dir / f"{connection_id}.json"
|
||||
|
||||
def save_auth(self, record: AuthRecord) -> None:
|
||||
self._auth_path(record.connection_id).write_text(
|
||||
json.dumps(asdict(record), indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def load_auth(self, connection_id: str) -> AuthRecord | None:
|
||||
path = self._auth_path(connection_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return AuthRecord(**data)
|
||||
|
||||
def save_catalog(self, snapshot: CatalogSnapshot) -> None:
|
||||
payload = {
|
||||
"connection_id": snapshot.connection_id,
|
||||
"fetched_at_epoch_ms": snapshot.fetched_at_epoch_ms,
|
||||
"max_age_seconds": snapshot.max_age_seconds,
|
||||
"nodes": [asdict(node) for node in snapshot.nodes],
|
||||
}
|
||||
self._catalog_path(snapshot.connection_id).write_text(
|
||||
json.dumps(payload, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def load_catalog(self, connection_id: str) -> CatalogSnapshot | None:
|
||||
path = self._catalog_path(connection_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return CatalogSnapshot(
|
||||
connection_id=data["connection_id"],
|
||||
fetched_at_epoch_ms=data["fetched_at_epoch_ms"],
|
||||
max_age_seconds=data["max_age_seconds"],
|
||||
nodes=[CatalogNodeEntry(**node) for node in data.get("nodes", [])],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user