From c2cfbdcb25db385b84558b36b288ae862a26cb6f Mon Sep 17 00:00:00 2001 From: lda Date: Fri, 11 Sep 2026 15:12:08 +0700 Subject: [PATCH] docs: revise thesis argument and verify Python walkthrough --- docs/thesis/references.bib | 80 ++ docs/thesis/system-design-implementation.md | 752 +++++++++++------- examples/report_workflow/wf.config.json | 7 + .../test_thesis_python_walkthrough.py | 76 ++ 4 files changed, 619 insertions(+), 296 deletions(-) create mode 100644 tests/examples/test_thesis_python_walkthrough.py diff --git a/docs/thesis/references.bib b/docs/thesis/references.bib index 1cb980a6..f481aa1c 100644 --- a/docs/thesis/references.bib +++ b/docs/thesis/references.bib @@ -6,6 +6,86 @@ urldate = {2026-09-07} } +@online{n8n-publish-2026, + title = {Save and publish workflows}, + author = {{n8n}}, + year = {2026}, + url = {https://docs.n8n.io/build/understand-workflows/save-and-publish-workflows.md}, + urldate = {2026-09-11} +} + +@online{n8n-executions-2026, + title = {View executions for a single workflow}, + author = {{n8n}}, + year = {2026}, + url = {https://docs.n8n.io/build/understand-workflows/understand-executions/view-executions-for-a-single-workflow.md}, + urldate = {2026-09-11} +} + +@online{zapier-versions-2026, + title = {Create Zap drafts and versions}, + author = {{Zapier}}, + year = {2026}, + url = {https://help.zapier.com/hc/en-us/articles/9693520498445-Create-Zap-drafts-and-versions}, + urldate = {2026-09-11} +} + +@online{zapier-history-2026, + title = {View and manage your Zap history}, + author = {{Zapier}}, + year = {2026}, + url = {https://help.zapier.com/hc/en-us/articles/8496291148685-View-and-manage-your-Zap-history}, + urldate = {2026-09-11} +} + +@online{zapier-connections-2026, + title = {Manage your app connections}, + author = {{Zapier}}, + year = {2026}, + url = {https://help.zapier.com/hc/en-us/articles/8496290788109-Manage-your-app-connections}, + urldate = {2026-09-11} +} + +@online{n8n-data-2026, + title = {Understand n8n's data structure}, + author = {{n8n}}, + year = {2026}, + url = {https://docs.n8n.io/build/work-with-data/understand-n8ns-data-structure.md}, + urldate = {2026-09-11} +} + +@online{n8n-order-2026, + title = {Understand execution order}, + author = {{n8n}}, + year = {2026}, + url = {https://docs.n8n.io/build/flow-logic/understand-execution-order.md}, + urldate = {2026-09-11} +} + +@online{n8n-wait-2026, + title = {Wait}, + author = {{n8n}}, + year = {2026}, + url = {https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.wait/}, + urldate = {2026-09-11} +} + +@online{zapier-mapping-2026, + title = {Send data between steps by mapping fields}, + author = {{Zapier}}, + year = {2026}, + url = {https://help.zapier.com/hc/en-us/articles/8496343026701-Send-data-between-steps-by-mapping-fields}, + urldate = {2026-09-11} +} + +@online{langgraph-interrupts-2026, + title = {Interrupts}, + author = {{LangChain}}, + year = {2026}, + url = {https://docs.langchain.com/oss/python/langgraph/interrupts}, + urldate = {2026-09-11} +} + @online{zapier-paths-2026, title = {Add branching logic to Zap workflows with Paths}, author = {{Zapier}}, diff --git a/docs/thesis/system-design-implementation.md b/docs/thesis/system-design-implementation.md index f812c0d3..6f142f9e 100644 --- a/docs/thesis/system-design-implementation.md +++ b/docs/thesis/system-design-implementation.md @@ -51,8 +51,8 @@ diagram: | Abbreviation | Meaning | | --- | --- | | API | Application Programming Interface | +| AI | Artificial Intelligence | | CLI | Command-Line Interface | -| DAG | Directed Acyclic Graph | | JSON-RPC | JavaScript Object Notation Remote Procedure Call | | LLM | Large Language Model | | MCP | Model Context Protocol | @@ -86,11 +86,9 @@ choice to separate a node's data output from its routing outcome. The implementation provides Python authoring objects, a workflow service, and adapters for trusted Python functions and external tools. Evidence includes -a deterministic case study and focused conformance tests. These establish -specific engineering results, not broad usability, model superiority, or -reduced token use. The system remains in -development, with further work needed on authoring ergonomics and user -experience. +a deterministic case study and focused tests of validation, +execution, and persistence. Authoring usability remains to be evaluated; +the prototype's interaction design is still under development. # Introduction @@ -133,32 +131,34 @@ leaves a consequential rule implicit. The current prototype emphasizes programmable authoring and inspection. Its Python interface presents workflow and run objects instead of requiring -authors to construct network messages. This supports code-based use, but does -not establish accessibility for non-programmers. Onboarding, edit feedback, -error presentation, and the effort needed to understand a workflow remain -areas for user-experience improvement. +authors to construct network messages. The interaction considered here is +therefore code-based: discovering operations, revising a graph, and inspecting +its executions through Python objects. ## Engineering question and contribution -The engineering question is how to make a reusable workspace procedure both -operable through public interfaces and explicit enough for the runtime to -validate, execute, and record. +This thesis examines how authors can define a reusable procedure while the +system manages its individual executions. Authors specify data connections +and decisions about what happens next. The system must preserve those rules +when steps repeat, when one workflow calls another, and when execution pauses +for input. +Separating these responsibilities allows a saved procedure to be executed +without retaining the conversation or programming session that produced it. -The contribution connects three design choices: +The prototype implements three design choices as one lifecycle: -1. A typed graph makes operations, data movement, and routing decisions - inspectable rather than leaving the procedure only in a conversation. +1. A graph with declared input, output, and state schemas represents operations, + data movement, and routing decisions so they can be validated and inspected. 2. Separate saved versions, environment bindings, and execution records make editing, configuration, and operation distinct activities. 3. Programmable authoring, validation diagnostics, and run inspection expose those distinctions to human and agent clients. -The work integrates established workflow techniques rather than introduces a -new autonomous planning algorithm. Its design is assessed from both sides: +The work integrates established workflow techniques into a service for reusable +procedures. Its design is examined from both sides: what an author must understand and do, and what the runtime guarantees when -the procedure executes. The comparison with related systems uses concrete -mechanisms rather than treating visual polish or graph notation as sufficient -evidence of either usability or correctness. +the procedure executes. The comparison with related systems examines concrete +authoring and execution mechanisms. ## Scope of the implementation @@ -170,11 +170,11 @@ tested behavior from unmeasured usability and production-readiness claims. ## Report Outline -Section 2 derives interaction and execution requirements from the workspace -task. Section 3 compares ways to author and operate that task in related -systems. Section 4 explains the prototype's concepts through an example. -Sections 5 and 6 describe the architecture and implementation; Section 7 presents -the reproducible case study. Section 8 evaluates the available evidence. +Chapter 2 derives interaction and execution requirements from the workspace +task. Chapter 3 compares ways to author and operate that task in related +systems. Chapter 4 explains the prototype's concepts through an example. +Chapters 5 and 6 describe architecture and implementation; Chapter 7 presents +the reproducible case study. Chapter 8 evaluates the available evidence. The remaining chapters discuss limitations, future work, and conclusions. # Problem Statement And Requirements @@ -186,54 +186,71 @@ needed by the final document. The author needs feedback that distinguishes these situations, while the runtime needs rules for handling them. LLM tool-use approaches illustrate dynamic selection of actions -[@react-2022; @toolformer-2023]. This thesis does not assume that such systems -cannot use schemas or persistence. It asks which responsibilities should be -explicit in the reusable procedure rather than depend on the authoring session. +[@react-2022; @toolformer-2023]. Such approaches can incorporate schemas and +persistence. This thesis examines which responsibilities belong in the saved +procedure and which remain with its authoring environment. + +The requirements are organized into two groups. R1–R5 describe the authoring +and operation experience the system should support. X1–X5 define the execution +rules needed to support that experience. Their identifiers link the design +to the evidence assessment in [@tbl:requirements-evidence]. ## Authoring and operation requirements -The following requirements describe what the interaction should support. -They are design goals, not claims that every aspect of the current experience -has been validated with users. +The authoring requirements cover discovery, data movement, revision, version +selection, and execution inspection: -1. **Discover before connecting.** Show available operations and the inputs, +1. **R1 — Discover before connecting.** Show available operations and the inputs, outputs, and services they require. The author should not need to inspect server implementation code to learn how an operation can be used. -2. **Make data movement understandable.** Explain how a document path becomes +2. **R2 — Make data movement understandable.** Explain how a document path becomes text, how text becomes structured fields, and which fields reach the result. Distinguish a data connection from a decision about what executes next. -3. **Support revision and useful feedback.** An author should be able to revise - an unfinished procedure and locate errors in the relevant step or binding. - Diagnostics should distinguish what must change from what remains valid. -4. **Separate editing from running.** Make clear which saved version an execution +3. **R3 — Support revision and useful feedback.** An author should be able to revise + an unfinished procedure and locate errors in the relevant step or data + mapping. Diagnostics should locate the rejected part and explain the + violated constraint. +4. **R4 — Separate editing from running.** Make clear which version an execution uses. Editing the next version should not silently change an earlier one. -5. **Explain execution state.** Distinguish completed, failed, and interrupted +5. **R5 — Explain execution state.** Distinguish completed, failed, and interrupted runs, expose relevant intermediate evidence, and identify the input required to resume an interruption. -For machine clients, these interactions need structured responses and stable -identities. For human authors, they also need understandable terminology and -manageable amounts of information. Providing structured output satisfies an -interface requirement; it is not proof that the overall experience is usable. +Agent-assisted authoring places particular demands on this interaction. An +agent must discover the available operations, interpret a rejected definition, +and determine which changes are permitted without relying on implementation +files. This motivates three complementary forms of support: descriptions of +operations and their contracts, diagnostics that locate errors, and instructions +for the authoring and execution lifecycle. Structured responses, stable +identities, and inspection results with explicit size limits make that information +available to +programmatic clients. Human authors and other software clients use the same +information to construct and operate workflows. + +A saved definition follows the same routing and data rules +whether it was written by a developer or assembled by an agent. The evaluation +therefore examines ease of use and compliance with execution rules as separate +questions. ## Execution requirements behind the interaction -Those interactions require corresponding runtime contracts: +Those interactions require corresponding runtime contracts (X1–X5): -1. **Preserve definitions and executions independently.** Save an identifiable +1. **X1 — Preserve definitions and executions independently.** Save an identifiable procedure version and keep separate records for its invocations. -2. **Validate known constraints before work starts.** Check graph structure, - declared data contracts, mappings, and required service bindings without - pretending to predict every external failure. -3. **Specify routing and state changes.** Define what selects the successor, - where outputs are written, and how repeated writes affect workflow state. -4. **Keep environment choices outside the procedure's logic.** Resolve logical +2. **X2 — Validate known constraints before work starts.** Check graph structure, + declared data contracts, mappings, and required service bindings. +3. **X3 — Specify routing and state changes.** Define what selects the successor, + where outputs are written, and how repeated writes affect the workflow's + working data, referred to here as workflow state. +4. **X4 — Separate environment choices from procedure logic.** Resolve logical service requirements to concrete configured services and report mismatches. -5. **Bound and inspect execution.** Limit execution attempts, retain status and - trace information, and support resume at defined interruption boundaries. +5. **X5 — Bound and inspect execution.** Limit execution attempts, retain + status and trace information, and support resume at defined interruption + boundaries. -These requirements motivate the artifact, deployment, and run distinctions. -They do not prescribe those names for every competing system. +Together, these requirements motivate separate records for saved procedures, +environment selection, and individual executions. ## Costs and boundaries @@ -246,7 +263,6 @@ break-even point. A workflow interface must also avoid promising more than its runtime supports. Successful validation does not guarantee a remote service will succeed, and a saved interruption does not make arbitrary external effects reversible. -Scheduling and general fork/gather remain separate development work. # Positioning And Related Systems @@ -255,7 +271,7 @@ notes between them, add a route for incomplete information, test the procedure, and inspect its result. The comparison follows these authoring activities into the execution rules they expose. -The accounts below use documented mechanisms consulted on September 7, 2026. +The accounts below use documented mechanisms consulted in September 2026. They are not hands-on usability measurements or a complete product survey. Differences in authoring style do not establish that one system is easier for every audience. @@ -267,7 +283,9 @@ language's conditionals and loops. An agent can instead select successive tool calls from the information available at each turn. Both approaches can be combined with schemas, tests, logs, and persistence. -Code puts the procedure close to its implementation and makes ordinary debugging +Scripts, including generated scripts, are a substantive alternative rather +than merely a preliminary form of workflow automation. Code puts the procedure +close to its implementation and makes ordinary debugging tools available. It also leaves decisions about configuration, saved versions, and run records to the program or its surrounding infrastructure. A workflow platform makes some of those decisions part of its public contract, at the cost @@ -278,26 +296,50 @@ code performs the entire procedure directly or constructs a saved workflow for the service to execute. Neither representation is automatically better for a one-off task. +For this comparison, state means working data retained during execution; +a reducer defines how a new write changes a state field. The prototype's +outcome is a routing label returned separately from a step's data output. +Chapter 4 develops these concepts through the report example. + +The comparison asks the same questions of each system: what a step produces, +how subsequent work is selected, how data combines, and how an author sees +those rules. Suspension is considered separately from ordinary branching. + ## n8n: connecting and inspecting data +n8n passes arrays of data items between connected nodes. Items contain JSON +data and may also contain binary data. Authors map fields from incoming items +into node parameters; dragging a field into a parameter creates an expression. +Thus, a connection participates in data flow as well as execution order +[@n8n-data-2026]. + For an author combining report records, n8n's Merge node exposes a concrete choice: append the incoming collections, match records by fields or position, or produce combinations. The node's configuration and worked examples make these operations distinguishable. Append waits for connected inputs and emits their items in input order [@n8n-merge-2026]. -This connects an interface decision to an execution rule. Selecting a merge mode -does not merely change a diagram: it changes which records appear in the output. +Selecting a merge mode changes which records appear in the output. For the report task, joining actions by owner is different from appending two action lists. -The lesson for the prototype is that a connection needs an understandable data -meaning. Its state reducers describe how writes update fields; they should not -be presented as interchangeable with an item join or synchronization barrier. -This comparison concerns the documented Merge mechanism, not every n8n node. +The prototype represents data movement through explicit mappings. Its reducers +define how writes update state fields. Joining records and waiting for branch +completion are separate operations. +Branching does not necessarily imply simultaneous execution. For workflows +created from n8n 1.0, the documented default completes one branch before +starting another, with ordering affected by canvas position and workflow +settings [@n8n-order-2026]. The author therefore needs both item-level data +inspection and an account of branch execution, not only a connected diagram. ## Zapier: configuring a decision +Zapier exposes the output fields of earlier steps for mapping into later +steps. Test records supply the values shown during configuration; live runs +use their own data. This makes the mapping an explicit reference to a previous +step, rather than an update to an author-declared shared state field +[@zapier-mapping-2026]. + An author using Zapier Paths selects fields, conditions, and values, then tests the rules against sample data. Applied to the report task, those rules could distinguish complete records from records needing attention. Multiple paths @@ -311,7 +353,7 @@ organizes a common report-rendering step [@zapier-paths-2026]. The interface therefore needs to communicate both the condition being tested and the consequence of a match. In the prototype, an ordinary outcome selects -one successor. That is a different contract, not a claim of superior usability. +one successor, whereas several Zapier Paths may qualify. ## LangGraph: describing decisions in Python @@ -333,21 +375,70 @@ decisions need a dedicated condition step or an additional declared outcome. The relevant comparison is where authors express and inspect decisions, not whether Python or a canvas is inherently the better interface. +## Saved Definitions, Configuration, and Execution Records + +Lifecycle separation is not unique to the prototype. n8n distinguishes saved +edits from the published version used for production execution, and separates +workflow history from execution history. Its execution view supports status +filtering and inspection of previous attempts +[@n8n-publish-2026; @n8n-executions-2026]. + +Zapier allows draft editing while a published Zap remains active. Publishing +creates a version, and run details identify the version used and the data +received and sent by individual steps. Connected application accounts are +managed separately through app connections +[@zapier-versions-2026; @zapier-history-2026; @zapier-connections-2026]. + +For LangGraph's library API, the graph is defined and compiled in application +code. Checkpointers store execution snapshots organized by thread identity; +the application can retrieve current state and state history. Application +configuration and dependency provision remain part of the surrounding code. +This library-level comparison does not cover hosted deployment products +[@langgraph-graph-api-2026; @langgraph-persistence-2026]. + +The prototype exposes saved definitions, source selection, and run inspection +as artifact, deployment, and run objects in its Python client. Its deployment +mapping selects a configured provider that must satisfy the saved source +requirements. An app connection or credential is therefore only a partial +analogy: the source also supplies operations and their contracts. + +The design contribution is the composition of these established lifecycle +responsibilities with the typed graph and client interface. Each system must +distinguish an edit to future work from the recorded +definition and data of a past execution. + ## Implications for this design -The examples expose three connected design concerns: authors must understand -the data exchanged, the condition selecting work, and the meaning of execution -progress. A graph drawing alone does not answer any of them. +[@tbl:positioning-summary] compares the documented mechanisms with the +prototype's ordinary execution model. Its rows describe selected mechanisms, +not every extension available in each product. The sources for n8n, Zapier, +and LangGraph are discussed in the preceding subsections. -| Author's question | Execution concept it exposes | -| --- | --- | -| Which result should the next step receive? | Input/output mapping | -| Will one branch run, or several? | Exclusive routing or concurrent emission | -| What happens where paths meet? | Continuation, data merge, or a barrier | -| Why did this attempt stop? | Failure versus explicit interruption | -| What does changing the workflow affect? | Saved version versus a run | +| System | Data | Control selection | Combination | +| --- | --- | --- | --- | +| n8n | Item arrays | Branch connections | Merge modes | +| Zapier | Prior-step fields | Matching Paths | Explicit later actions | +| LangGraph | State updates | Edges and routers | Field reducers | +| Prototype | Output-to-state mappings | One outcome edge | Field reducers | -: Interaction and execution concepts. {#tbl:positioning-summary} +: Data and control mechanisms compared. {#tbl:positioning-summary} + +For the report task, these models put different work on the author. n8n +requires attention to which items reach each node; Zapier requires mappings +from earlier steps and rules for qualifying paths; LangGraph requires state +and routing code. The prototype instead requires explicit output-to-state +mappings and declared routing outcomes. These differences concern where the +procedure's meaning is expressed, not just whether its editor is visual. + +Pausing also has a separate contract. n8n's Wait node can resume on a time, +webhook, or form condition [@n8n-wait-2026]. LangGraph's dynamic interrupt +uses a checkpoint and thread identity; resuming restarts the interrupted node, +so code preceding the interrupt executes again [@langgraph-interrupts-2026]. +The prototype uses an explicit interruption boundary and a declared resume +payload. A failed operation is not automatically such a pause: for example, +Zapier documents that an errored step produces no output fields for subsequent +mappings [@zapier-mapping-2026]. These observations do not establish equivalent +retry or side-effect guarantees across the systems. The prototype prioritizes programmable authoring, explicit mappings, and saved execution records. Its current interface must still be evaluated for the work @@ -357,22 +448,24 @@ that it achieves those user-experience goals. External-tool protocols are a separate concern. Model Context Protocol (MCP) exposes tools, resources, and prompts; it is not a competing graph model -[@mcp-tools-2025; @mcp-lifecycle-2025]. Here MCP is a source family, not the -product identity. Protocol details belong later, after the authoring and -execution concepts they support have been explained. +[@mcp-tools-2025; @mcp-lifecycle-2025]. In this system, a source is a configured +provider of operations. MCP is one way to obtain those operations; it does not +determine the workflow's routing or data model. # Conceptual Model The report example introduces the concepts in the order an author encounters them: choose operations, connect their data, define decisions, save a version, -and inspect an execution. The following branching example explains supported +and inspect an execution. The branching example in [@fig:report-branch] +explains supported primitives; it is not an additional measured case study. ## Operations, data, and decisions Suppose report preparation must ask for missing information before rendering: -```{.mermaid #fig:report-branch width=95% caption="Report routes."} + +```{.mermaid #fig:report-branch width=95% caption="Alternative routes to report rendering."} %%{init: {"flowchart": {"rankSpacing": 20, "nodeSpacing": 20}}}%% flowchart LR Read["Read
notes"] --> Extract["Extract
report"] @@ -393,14 +486,17 @@ The arrow to the next step does not implicitly pass all previous output into that step. The author defines input mappings and writes relevant outputs into workflow state. In this example, extraction writes report fields, the request can supply missing fields on resume, and rendering reads the resulting report. -The two routes to rendering are alternatives, not concurrent branches needing -a join. +Only one of the two routes to rendering executes on each decision. **Schemas** declare the shapes of accepted inputs and produced results. They help the author see which fields an operation requires and allow the validator to detect incompatible mappings. They do not establish that an extracted fact is true or that a remote operation will succeed. +A node output is the data returned by one step. The workflow output is the +public data selected when the procedure completes. A run records that workflow +output alongside status, diagnostics, execution identity, and trace information. + ## Workflow state and iteration **State** is the workflow's working data. A **reducer** specifies how a write @@ -410,12 +506,14 @@ operations. This makes data movement inspectable but requires the author to understand the mappings. If the procedure processes several documents, a foreach step defines an item -body. The current item and its iteration context belong to that body; normal -item completion returns to the owning foreach. A child workflow provides a +body. The foreach distinguishes completion of one item from completion of the +collection. A child workflow provides a separate invocation scope with explicit inputs and results. These boundaries define which data is available and what completion means. -Ordinary same-region cycles are permitted, with a run-wide step budget limiting +Nodes outside an iteration body, or within the same body, may form ordinary +cycles. A body and its enclosing workflow are separate control regions; +an ordinary edge cannot cross that boundary. A run-wide step budget limits execution attempts. General fork/gather is not implemented. Connecting paths does not by itself promise parallel execution, synchronization, or conflict-free state merging. @@ -423,8 +521,7 @@ state merging. ## Saving, configuring, and running the procedure While authoring, an **editable workflow** is the mutable definition being -revised. The Python client supports this object without requiring a server-side -draft workspace. Saving creates an **artifact**: an immutable workflow version +revised. Saving creates an **artifact**: an immutable workflow version containing its graph and declared requirements. A **deployment** selects a saved version and connects its logical service @@ -439,9 +536,12 @@ same artifact and deployment. Inspecting a run should identify the version, inputs, status, and available execution evidence without changing the saved procedure. -These distinctions explain the lifecycle from the operator's perspective: +[@fig:lifecycle-records] relates the saved version, its deployments, and their +runs. The separate records preserve the distinction between changing a +procedure and examining an execution of it. -```{.mermaid #fig:lifecycle-records width=95% caption="Lifecycle records."} + +```{.mermaid #fig:lifecycle-records width=95% caption="One saved version can serve several deployments and runs."} classDiagram direction LR class EditableWorkflow { @@ -473,7 +573,8 @@ A **source** groups capabilities under a configured identity. The report's extraction operation might come from trusted Python code, while another operation is supplied by an external tool service. -A deployment **binding** connects a logical source requirement in the workflow +A **binding** is an explicit mapping. Input and output bindings map data; +a deployment binding connects a logical source requirement in the workflow to a concrete source in the environment. Validation checks whether that source exists and matches the saved requirements. **Source drift** means those requirements no longer match the currently available capabilities, for example @@ -481,8 +582,8 @@ after an input schema changes. Built-in sources have fixed platform identities and do not require those deployment bindings. Configured sources remain explicit operator choices. -This provides scoped portability, not freedom from environment dependencies: -the required code, credentials, and services must still be available. +Portability is limited to environments with compatible code, credentials, +and services. ## Inspecting failure and resuming an interruption @@ -504,7 +605,7 @@ these concepts become runtime records and service operations. ## Working Glossary -The core terms can now be summarized without requiring implementation vocabulary. +[@tbl:working-glossary] summarizes the core terms through the report example. | Term | Meaning in the report example | | --- | --- | @@ -517,7 +618,7 @@ The core terms can now be summarized without requiring implementation vocabulary | Deployment | A saved version connected to concrete services | | Run | One execution with its own status and evidence | | Source | A configured collection of available operations | -| Binding | A logical source requirement mapped to a concrete source | +| Binding | An explicit data mapping or source-to-environment mapping | : Working glossary for the thesis terminology. {#tbl:working-glossary} @@ -558,7 +659,8 @@ against the service's contracts. [@fig:architecture-spine] shows this separation. The CLI is another entry point to the service; it is not a mandatory layer beneath Python authoring. -```{.mermaid #fig:architecture-spine caption="Shared service boundaries."} + +```{.mermaid #fig:architecture-spine caption="Clients share lifecycle services and provider-independent execution."} flowchart TB subgraph Authoring["Authoring and clients"] Python["Python App and editable workflow"] @@ -605,22 +707,20 @@ author has explicitly included an operation that makes such a decision. ## Keeping a Definition Separate from Its Use -Saving, deploying, and running answer different questions: +The client owns the editable graph; the service owns saved artifact, +deployment, and run records. The API coordinates validation and persistence +through their respective stores. Execution reads a selected definition and +its resolved environment rather than the client's mutable editor contents. -- An artifact identifies the saved workflow definition and its version. -- A deployment selects that version and supplies environment bindings. -- A run records one execution, including its input, status, and progress. - -For the report workflow, changing the extraction step produces a different -definition. Choosing the configured service that supplies extraction is an -environment decision. Processing this week's notes is an individual run. -Keeping these apart allows an operator to inspect which definition and -bindings an execution used without confusing them with the current editor -contents. +For the report workflow, this boundary prevents an unfinished edit to the +extraction step from becoming the definition of an already-created run. +Inspection retrieves the run's recorded identity and execution evidence; +it does not reconstruct the attempt from whatever is currently open in the +author's session. The client exposes this progression through workflow artifact, deployment, -and run objects. A run object is a snapshot, not a live background -subscription. Refreshing it requests a new snapshot. This makes network +and run objects. A run object contains a snapshot of the stored execution. +Refreshing it requests a new snapshot. This makes network activity explicit, although applications must decide when to refresh and how to present progress. @@ -641,7 +741,8 @@ routing to a renderer does not, by itself, supply the report it needs. iteration controllers, subgraphs, and interrupts have their own runtime handlers rather than pretending to be remote capability calls. -```{.mermaid #fig:node-execution-cycle caption="Callable step execution."} + +```{.mermaid #fig:node-execution-cycle caption="A step updates state before following its selected route."} flowchart TB Validation["Input validation"] --> Call["Invoke operation"] Call --> Result["Checked result"] @@ -665,8 +766,8 @@ being a failed runtime execution. Where iteration supports collecting item errors, that policy must be explicit; errors do not automatically become ordinary outcome edges. -The lanes distinguish responsibilities, not concurrent tasks: the runtime -applies the result's writes before advancing along the selected route. +The runtime performs the illustrated operations in sequence: it applies the +result's writes before advancing along the selected route. The trace makes the sequence inspectable, but a fixed graph does not imply identical external results. Language-model calls, remote services, and @@ -691,7 +792,8 @@ It does not begin the collection again whenever an item returns. downward for time. Solid arrows request work or apply a data binding; dashed arrows report completion. The labels identify the data and routes. -```{.mermaid #fig:foreach-region width=95% caption="Two documents, one loop."} + +```{.mermaid #fig:foreach-region width=95% caption="Serial iteration finishes both documents before assembly."} sequenceDiagram participant Each as Foreach participant Body as Render item @@ -710,11 +812,9 @@ sequenceDiagram State-->>Assemble: [reportA, reportB] ``` -The same authored render node executes twice, but the runtime must remember -which document each execution belongs to. It calls that per-item execution -record a **frame**. The foreach invocation that started the item is its -**owner**. Returning to that owner finishes the current item; arriving from -the preceding workflow step starts a new foreach invocation. +The same authored render node executes twice. The runtime distinguishes +those executions so that finishing A advances to B, while finishing B permits +assembly. Each item returns to the foreach invocation that started it. This explains the graph's back-edge: `render.ok -> each` is an item return, whereas `each.done -> assemble` leaves the loop. The output binding is what @@ -737,7 +837,8 @@ called it. If it needs another value, the author must add an input binding. the child runs. The child's `END` completes that child invocation, not the parent item or the whole report workflow. -```{.mermaid #fig:scope-boundaries width=95% caption="A child call returns."} + +```{.mermaid #fig:scope-boundaries width=95% caption="Child completion precedes the parent item's return to foreach."} sequenceDiagram participant Item as Parent item A participant Child as Child workflow @@ -755,10 +856,8 @@ sequenceDiagram Note over Each: Continue according
to foreach mode ``` -The implementation calls the child's isolated data environment a **scope**. -Its input binding crosses into that scope; its output binding maps the -returned result into the caller item's pending writes, or directly into -enclosing state in serial mode. The two completion points are separate: +Input and output bindings cross the child workflow boundary explicitly. +The two completion points are separate: child `END` returns to the calling node, and the calling node's route back to foreach finishes the item. @@ -768,18 +867,16 @@ In concurrent mode, A and B may be in progress together. A returning from its child does not permit assembly while B is still running. The foreach waits for its required item completions before following `done`. -Each item keeps its own pending writes, rather than immediately exposing -them to its sibling. The runtime tracks that separate state history as a -**lineage**; the pending writes are its **buffer**. In the second diagram, -“Item A reports writes” is that buffer when the foreach is concurrent. In serial -mode, the binding instead updates the enclosing state so the next item can -read it. +Concurrent items retain separate pending writes until their results combine. +In [@fig:scope-boundaries], “Item A reports writes” represents these pending +writes. In serial mode, the output binding updates enclosing state so the next +item can read it. For the successful two-item concurrent case, the foreach combines A's and B's writes using the declared reducers, then assembly reads the combined state. An append reducer and a replace reducer have different effects; neither the arrows nor the fact that both items completed chooses a merge -policy. The serial ordering shown in the first diagram is not a promise +policy. The serial ordering in [@fig:foreach-region] is not a promise about concurrent completion order. If a child or item reaches an explicit interrupt, the run must preserve @@ -809,9 +906,8 @@ another operation with the same fields. # Implementation -The implementation follows these boundaries through focused Python -packages. The important question is not the number of packages, but where a -change must be made. Improving a Python editing method should not require +The Python package structure localizes changes to the relevant responsibility. +Improving a Python editing method should not require changing the scheduler, and adapting a new remote service should not require changing graph routing. @@ -841,9 +937,8 @@ server still validates the submitted plan against its inventory. Saving validates first, submits the plan, checks the save response, and re-inspects the exact artifact version. The resulting artifact object is -therefore reconstructed from the stored definition, not assumed to be an -unchanged copy of the editor. Identity checks reject mismatched responses -instead of quietly attaching methods to the wrong artifact or deployment. +therefore reconstructed from the stored definition. Identity checks reject +responses identifying a different artifact or deployment. The same principle applies to runs. In `wf_client/runs.py`, `refresh()` returns a decoded snapshot from `inspect_run`, checking the expected run @@ -867,8 +962,7 @@ Step admission happens before dispatch. The immutable `RunLimits` policy sets a positive maximum, with a default of 10,000 attempts. The run stores how many attempts have been admitted. Nested execution shares that run-wide budget, and asynchronous dispatch reserves attempts before launching work. -A failed attempted step is not free simply because it returned no useful -result. +Failed step attempts also consume the budget. The budget bounds graph progress, including a cycle whose condition never selects an exit. It is not a wall-clock timeout, a language-model token @@ -878,12 +972,21 @@ does not reset the allowance. ## Preserving State Across Nested Execution +The runtime represents an individual execution position as a **frame**. +An item frame identifies its owning foreach invocation, distinguishing a +return from an item from a new entry into the controller. A **scope** contains +the input, state, and context of one workflow invocation. Child workflows +receive their own scopes. A **lineage** records a separate state history; +its buffered writes keep concurrent items isolated until merging. + Workflow state updates pass through reducer-aware patches. Serial iteration must make its writes visible to subsequent serial work, while concurrent items need separate views until their results are combined. Nesting either mode inside the other makes write ownership more subtle than committing every result directly to global state. +Two rules govern this routing: serial work observes preceding serial +writes, and concurrent siblings do not observe each other's unmerged writes. The shared `commit_foreach_aware_patch` helper handles writes from ordinary nodes, subgraph results, and interrupt responses. It walks serial owners outward and selects the first concurrent item boundary, if present, as the @@ -892,6 +995,12 @@ missing parent or cycle cannot cause a partial write merely because a buffer destination was already found. With no concurrent boundary, the patch commits through the enclosing serial owners. +Combining results must also preserve each write's contribution, rather than +count a previously merged value as new work. For example, if an enclosing +list already contains `A` and an inner group appends `B` and `C`, its visible +result is `[A, B, C]`, but its contribution is only `[B, C]`. Appending the +visible result again would duplicate `A`. + When concurrent results are combined, the patch retains their constituent write contributions for later reducer replay. Keeping only cumulative values would allow a surrounding iteration to replay an already-counted @@ -899,8 +1008,8 @@ prefix. This distinction matters for operations such as appending report sections: a correct visible value at one nesting level is not necessarily a correct contribution to the next merge. -This machinery currently supports iteration. It should not be read as a -claim that arbitrary graph fork/gather semantics are already implemented. +This implementation supports iteration; general fork/gather remains +future work. ## Giving Expressions a Consistent Context @@ -908,18 +1017,18 @@ Bindings and conditions need the same account of the current execution. The runtime's `frame_context_view` derives structured foreach entries from persisted frame ancestry. Entries are keyed by foreach node identity, so nested bodies can refer to enclosing items within the same workflow scope, -rather than relying only on an innermost-item shortcut. +as well as the current item. The walk stops at a subgraph scope boundary. An enclosing item's value must be passed as child input if the child needs it. The reader also rejects malformed ownership, parent cycles, and conflicting aliases; corrupt checkpoint metadata is not treated as an innocently absent field. -Condition evaluation receives this structured mapping, as do the input -resolution paths. This connection is necessary for validation to mean -anything: accepting a context path while evaluating conditions against a -smaller stub would let a valid-looking graph silently choose the wrong -branch. +A context path must have the same meaning in a condition and +an input binding within one execution. Both receive the structured mapping, +while validation checks paths against the corresponding context schema. +Otherwise, a condition testing whether the current item exists could select +the false route even though an input binding can read that item. ## Validation, Persistence, and Diagnostics @@ -953,7 +1062,7 @@ what happened during a particular attempt. The remaining package boundaries put these operations into a service. `wf_api` coordinates lifecycle operations, `wf_artifacts` supplies storage contracts and implementations, and `wf_platform` supplies shared platform -contracts. `wf_server` composes these dependencies. `wf_transport` exposes +contracts. `wf_server` composes these dependencies. `wf_transport_rpc_http` exposes the JSON-RPC interface, while `wf_cli` provides terminal operations. Provider implementations retain their own lifecycle requirements. MCP @@ -963,12 +1072,8 @@ a sandbox for arbitrary submitted code. OpenAPI support remains experimental and is not evidence that every described HTTP service can already be used without adaptation. -These seams make additional interfaces possible, but an interface still -needs its own interaction design. The availability of API operations and -typed client objects does not, by itself, establish that workflow authoring, -diagnosis, or recovery is easy for a new user. The case study and evaluation -therefore need to distinguish demonstrated operations from broader -usability claims. +The case study follows these components through one public authoring session: +discovery, revision, validation, persistence, execution, and inspection. # Case Study: Deterministic Report Workflow @@ -993,12 +1098,20 @@ who need it; that route is not a prerequisite for using this interface. ## Starting with Available Operations +The operator supplies the source identities through the example configuration. +This walkthrough discovers operations within those known sources; source +administration is outside the session. + The example configuration registers three trusted Python operations under -`local.report`. Their Pydantic models describe the input and output +`local.report` for discovery and `local.report_runtime` for execution. +Their Pydantic models describe the input and output contracts. The author consumes those operations from the service inventory; the client does not import their implementations to execute them locally. -The following blocks form one asynchronous Python session. They assume a +The following blocks form one asynchronous Python session. The retained test +executes these blocks against the real service in process, replacing only +the HTTP connection assignment. It does not verify remote server startup. +For HTTP use, the blocks assume a server using `examples/report_workflow/wf.config.json`, reachable at its configured address, and a fresh artifact name or unused version. The fixture read assumes the client is running from the repository root. @@ -1008,25 +1121,32 @@ from pathlib import Path from pydantic import BaseModel -from examples.report_workflow.ops import ReportOutput from wf_authoring import input_from, input_path, output_to, state_path from wf_client import App app = App.from_http_jsonrpc("http://127.0.0.1:8771/rpc") +available = await app.capabilities(source_id="local.report") +for operation in available.items: + print(operation.qualified_name) read_notes = await app.capability("local.report.read_notes") extract_report = await app.capability("local.report.extract_report") render_report = await app.capability("local.report.render_markdown_report") +print(extract_report.output_schema) ``` Each lookup returns a capability object with its schemas and declared -outcomes. Importing `ReportOutput` above only reuses the fixture's data model -for authoring; the three capability objects still refer to server-side -operations. An application without that shared model could use the inspected -JSON Schemas instead. +outcomes. The extraction schema exposes `title`, `summary`, `action_items`, +`risks`, and `followups`; each action item has `owner`, `task`, and `due` fields. +The author can use the schema directly or declare corresponding local models. +This session uses local models without importing provider implementation code. -This discovery step exposes a practical requirement: authors need to know -which operations exist and what data they accept before connecting them. -A name alone is not enough to establish a compatible pipeline. +The listing contains `local.report.read_notes`, `local.report.extract_report`, +and `local.report.render_markdown_report`. It establishes the available names; +the inspected contracts then describe how each can be connected. The author +still selects the operations, rather than the service synthesizing a plan. + +Before connecting operations, the author checks their input and output +contracts for compatible fields. ## Describing the Workflow's Data @@ -1034,6 +1154,24 @@ The workflow has one public input, intermediate state, and two public outputs. They are declared separately so that intermediate notes do not accidentally become part of the result contract. +```python +class ActionItem(BaseModel): + owner: str + task: str + due: str + + +class ReportOutput(BaseModel): + title: str + summary: str + action_items: list[ActionItem] + risks: list[str] + followups: list[str] +``` + +These author-defined models represent the discovered report fields. The +workflow then declares its own input, working state, and public output: + ```python class NotesInput(BaseModel): text: str @@ -1104,11 +1242,10 @@ whole output object. The other output bindings select individual fields. The `connect` calls then specify execution order for the `ok` outcome; they do not implicitly carry those objects between steps. -This explicitness is both a benefit and an authoring cost. The mapping is -inspectable, and changing a route does not silently change a data source. -However, even a linear three-step procedure requires contracts, bindings, -and routes. Typed helpers reduce raw serialization work without removing -the need to understand these distinctions. +The author declares data mappings and execution routes separately. Changing +one leaves the other unchanged. This requires additional declarations even +for a linear three-step procedure; typed helpers construct the serialized +representation from those declarations. ## Diagnosing and Repairing a Binding @@ -1133,9 +1270,17 @@ graph.set_output([ (await graph.validate()).raise_for_errors() ``` -The local report identifies the invalid source path in the workflow's output -projection. No server validation request is made for that invalid graph. -The repair changes the projection, not the renderer or its outgoing edge. +The diagnostic has code `invalid_source_path` and location `output[0].path`. +Its message is: + +> source path must start with input., state., or context. and reference a +> declared root field when applicable + +This locates the rejected mapping but does not name a replacement field; +the author must compare it with the declared state. No server validation +request is made for that invalid graph. +The repair changes the workflow's output mapping. The renderer and its +outgoing edge remain unchanged. This illustrates why data bindings and control routes need separate feedback. The next section saves only the repaired definition. @@ -1151,7 +1296,7 @@ artifact = await graph.save(version=1) deployment = await artifact.deploy( "report_python_showcase.local", - bindings={"local.report": "local.report"}, + bindings={"local.report": "local.report_runtime"}, ) readiness = await deployment.validate() if not readiness.runnable: @@ -1159,14 +1304,16 @@ if not readiness.runnable: ``` The artifact is the saved version of the authored procedure. The deployment -binds its logical source requirement to the configured source. Both happen -to be called `local.report` here; the mapping still records an environment -choice rather than a new graph edge. +maps the saved `local.report` requirement to `local.report_runtime`. +The fixture registers compatible operations under both names so this choice +changes the execution source without changing the graph or requiring remote +credentials. It demonstrates rebinding, not migration between real services. -The sequence below summarizes the public operations; it omits internal +[@fig:python-lifecycle] summarizes the public operations; it omits internal validation and re-inspection calls made by individual client methods. -```{.mermaid #fig:python-lifecycle caption="Authoring through the service."} + +```{.mermaid #fig:python-lifecycle caption="Local editing leads to saved, configured, and inspected execution."} sequenceDiagram actor Author participant Client as Python client @@ -1227,12 +1374,16 @@ unstructured notes. ## What This Case Demonstrates +The executable check in +[`test_thesis_python_walkthrough.py`](../../tests/examples/test_thesis_python_walkthrough.py) +reads and runs this chapter's Python blocks, including the rejected binding +and its repair. It checks the stored report as well as the returned snapshot. The existing tests in [`test_report_workflow_example.py`](../../tests/examples/test_report_workflow_example.py) check the source's input rules, rendering and extraction, capability discovery and invocation, and the artifact/deployment/run lifecycle using the raw-plan fixture. Those tests are evidence for the report operations -and lifecycle. They are not a user study of the Python walkthrough. +and lifecycle. They complement the Python session check; neither is a user study. The Python presentation makes the current authoring experience concrete: inspect operations, declare contracts, connect data and outcomes, save, @@ -1241,7 +1392,7 @@ another way to perform related lifecycle operations; they are not required steps in this Python walkthrough. A direct Python script would be shorter for these three local functions. -The workflow system earns its additional structure when the definition +The additional structure becomes relevant when the definition must be saved, bound to an environment, validated independently, and inspected through a shared service. This example demonstrates that integration, not a performance advantage over function calls. @@ -1257,65 +1408,89 @@ also requires evidence beyond a successful fixture run. Evaluation distinguishes three questions: whether the runtime follows its contract, whether the public lifecycle composes correctly, and whether an author can use that lifecycle effectively. The current evidence addresses -the first two through controlled tests and an executable walkthrough. -Evaluation of the intended shell-backed authoring experience remains pending. +the first two through controlled tests and an adapted in-process walkthrough. +The authoring assessment identifies the operations available through the +interface. Usability remains a separate evaluation question. -## Prototype Conformance Criteria +## Requirements and Evidence -The implementation should preserve definition and execution identity, reject -invalid structures and bindings, apply declared state updates, and expose -stopped runs for inspection or explicit resume. It should also allow -configured source families to supply operations without making the graph -scheduler specific to one remote protocol. +[@tbl:requirements-evidence] relates the requirements to the available +evidence. The walkthrough demonstrates interface operations supporting R1–R5. +The repository also contains tests covering X1–X5. The table distinguishes +these sources from the usability measurements still needed. -These are conformance criteria, not claims of general reliability. -The evidence index gives the source and test paths behind each row below. -A referenced suite identifies where a behavior is tested; it does not imply -that every suite was rerun during this document revision. - -| Claim | Evidence | Boundary | +| Requirement | Evidence | Assessment | | --- | --- | --- | -| Separate versions and runs | Lifecycle; E1 | No universal portability | -| Python lifecycle objects | Walkthrough; E2 | Not a usability study | -| Invalid bindings diagnosed | Validation; E3 | Not business correctness | -| Nested execution ownership | Runtime; E4 | No general fork/gather | -| Persisted step limits | Budget; E4 | Not a handler timeout | -| Provider-supplied operations | Sources; E5 | Unequal provider features | -| Explicit interrupt resume | Resume; E1 | Not arbitrary crash replay | +| R1 Discovery | Walkthrough; E2 | Exercised; usability unmeasured | +| R2 Data movement | Worked graph | Explained; comprehension unmeasured | +| R3 Revision feedback | Binding repair; E3 | Exercised; usability unmeasured | +| R4 Editing vs running | Lifecycle; E1–E2 | One saved version exercised | +| R5 Status interpretation | Inspection; E1–E2 | Exposed; usability unmeasured | +| X1 Definition/run identity | Lifecycle; E1 | Lifecycle tests listed | +| X2 Known constraints | Validation; E3 | Constraint tests listed | +| X3 Routing and state | Runtime; E4 | Runtime tests listed | +| X4 Environment choices | Sources; E5 | Provider tests listed | +| X5 Bounds and inspection | Budget/resume; E1, E4 | Boundary tests listed | -: Claims, evidence, and boundaries. {#tbl:prototype-conformance} +: Requirements and available evidence. {#tbl:requirements-evidence} -## Current Walkthrough Check +The R4 observation covers saving and running version 1. The session does not +edit a later version or check its effect on earlier runs. Understanding the +version distinction remains part of the authoring evaluation. -During this revision on September 7, 2026, an adapted version of the Python -blocks in the report walkthrough was executed as an in-process smoke check -against the example server configuration with an isolated temporary store. -The adapted check used an in-process client port in place of the displayed -HTTP connection; the walkthrough blocks were not executed unchanged. It -exercised discovery, graph construction, validation, saving, deployment, -execution, refresh, and trace inspection. +X5 contains three distinct obligations: admission limits, execution +inspection, and interruption resume. Budget tests exercise limit persistence +and exhaustion; lifecycle tests exercise stored inspection and resume. +The report session exercises inspection but does not exhaust its budget or +interrupt. The evidence index locates these separate checks; it is not a +record of a newly executed full-system suite. -The fixed fixture produced the expected title, three action items, and -Markdown heading. A deliberate invalid output binding was rejected before -saving; restoring the projection allowed validation and execution to proceed. -This adds a repair example without claiming that a new user would find the -diagnostic sufficient. +## Walkthrough Method and Observations -The check does not exercise network startup or HTTP transport, and is not a -performance benchmark. The documentation and report-example test files are -also run together as a focused verification target: +The retained walkthrough test uses the example server configuration and a +fresh pytest temporary store. It extracts the case study's Python blocks in +document order and substitutes an in-process client connection for the HTTP +connection assignment. All later calls use the real API, provider, and stores. +This setup exercises discovery, graph construction, validation, saving, +deployment, execution, +refresh, and trace inspection, while excluding network startup and transport +behavior from the observation. + +[@tbl:prototype-conformance] records the concrete checks performed by that +session. The fixture expectations are fixed independently of the workflow's +result, and stored output is inspected again through the API. + +| Check | Expected observation | Requirement | +| --- | --- | --- | +| Invalid output mapping | Local rejection at `output[0].path` | R3, X2 | +| Repaired graph | Validation permits saving and running | R3, X2 | +| Persisted report | Expected title and three action items | X1, X3 | +| Persisted rendering | Expected Markdown heading | X3 | +| Run and trace | Completed snapshot; recorded steps | R5, X5 inspection | + +: Reproducible report-session checks. {#tbl:prototype-conformance} + +The checked session completed with the expected title, three action items, +and Markdown heading in the saved output. The invalid binding was rejected +locally; restoring the mapping allowed validation and execution to continue. +These observations establish the displayed procedure's behavior for this +fixture, not an author's ability to construct it unaided. + +The session check and the existing report-example tests can be run with: ```powershell $suites = @( - "tests/docs/test_big_doc_links.py", + "tests/examples/test_thesis_python_walkthrough.py", "tests/examples/test_report_workflow_example.py" ) uv run pytest @suites -q -n 0 ``` -Those tests cover document integration and the existing raw-plan fixture. -They do not independently execute every Python block in the thesis; the -walkthrough execution is a separate editorial smoke check. +The first suite executes the displayed case-study session with the stated +transport substitution. The second checks the operation fixtures and raw-plan +lifecycle independently. Document rendering and generated PDF assets are not +part of this command. The test and manuscript must be taken from the same +repository revision because the test reads the manuscript directly. ## Separating Design Comparison from Evaluation @@ -1336,22 +1511,6 @@ users. Similarly, structured diagnostics and inspection objects may help an agent avoid trial and error, but reduced retries, token use, and repair time remain hypotheses rather than measured outcomes. -## Current Evaluation Boundaries - -A new agent evaluation should target the actual application-facing tools, -including the intended shell-backed workflow interface once integrated. -Its allowed operations, environment, task fixtures, and success criteria -must be fixed before collecting results. An agent's self-report should be -checked against saved artifacts, deployment identity, run output, and -recorded interactions. - -The earlier CLI evaluation is retained separately in -[repository history](../historical/thesis/2026-09-07-retired-agent-evaluation.md). -It is not included as evidence for the current authoring experience, and -there are no replacement campaign results in this thesis. A demonstration -of the surrounding application will not by itself establish authoring -success across tasks or users. - ## Falsifiability Criteria The implementation would fail its stated contracts if, for example: @@ -1390,8 +1549,6 @@ users can make these distinctions without assistance. Inspection also requires judgment. A trace shows recorded execution, not whether a report is factually correct or a remote side effect was desirable. -The system remains in development, and its authoring and operational -experience needs evaluation and refinement. ## Execution Guarantees Have Defined Boundaries @@ -1435,41 +1592,39 @@ would still need to preserve the lifecycle's transaction and ownership contracts; changing the storage engine alone would not prove those properties. -Scheduled deployment execution is implemented within the documented first -slice. It remains bounded by the filesystem-backed, single-process store and -the explicitly enabled local/static server composition. Nor does exposing a -provider's callable operations imply that its interactive widgets or entire -user experience are reproduced through the workflow API. +Scheduled execution uses the same deployment and run records as direct +invocation. Its ownership model assumes one scheduler over the participating +file stores and is supported only by the local/static server configuration. +This bounds the deployment conditions under which the scheduling mechanism +can be used; it is not a distributed execution service. + +The provider interface exposes callable operations. It does not reproduce a +provider's interactive widgets or its complete user interface. ## Limits of the Evidence The deterministic report fixture demonstrates lifecycle integration, not broad document understanding or graph expressiveness. Targeted tests cover -additional execution mechanisms, but their passing results apply to the -cases and revisions tested. +additional execution mechanisms. The evidence index identifies those tests; +this report records execution results for the report-session suites. -The current walkthrough check verifies documented calls against an isolated -service API, not network deployment or usability for an independent author. -An evaluation of the intended shell-backed application remains pending. - -There is no matched cross-system experiment or broad human user study. -Consequently, this report cannot claim that the prototype is easier to use, -more reliable, or faster than the systems discussed earlier. The comparison -explains design choices; the implementation evidence tests this system's -own behavior. +The in-process setup also limits the walkthrough to service composition; it +does not test the displayed HTTP connection. Without independent authoring +tasks or matched cross-system measurements, its results cannot establish +ease of use, repair efficiency, or comparative performance. Those questions +require the interaction evaluation described in Future Work. # Future Work -The next work should strengthen execution semantics without losing sight -of the author who must understand them. The live roadmap records engineering -order; the priorities below explain why that work matters to this design. +The remaining questions concern richer execution semantics, the effectiveness +of authoring and diagnosis, and operation beyond the controlled environment. ## Establish General Fork and Gather Semantics -The immediate runtime direction is to consolidate identity resolution and -then establish explicit fork/gather behavior. Existing frames, scopes, -iteration activations, and state lineages provide foundations, but their -relationships must remain coherent through nested execution and resume. +General fork/gather requires a rule for identifying which concurrent work +belongs to the same invocation. Existing frames, scopes, iteration activations, +and state lineages provide foundations, but their relationships must remain +coherent through nested execution and resume. A fork creates concurrent execution branches; a gather must determine which arriving branches belong together before combining their state. @@ -1499,37 +1654,41 @@ Conversely, a convenient editing operation must not hide a change to the workflow's execution meaning. The Python client and CLI should receive evidence appropriate to their own -interaction styles. Broader agent trials can vary tasks and instruction -profiles, while human evaluation can test whether the lifecycle vocabulary -and data-binding model are understandable without implementation knowledge. +interaction styles. For agent trials, fix the interface, allowed operations, +task fixtures, and success criteria before collecting results. Check an +agent's report against saved artifacts, deployment identity, run output, and +recorded interactions. Human evaluation can test whether the lifecycle +vocabulary and data-binding model are understandable without implementation +knowledge. -## Scheduling and the Surrounding Application +## Durable Waiting Within a Run -Scheduled deployment execution is implemented for the current slice. It -introduces trigger identity, overlap policy, and recovery decisions in -addition to time-expression parsing, and builds on the same run lifecycle -rather than creating a separate execution model. Extending it to distributed -workers or suspending an already-running workflow until a time or event is a -related but distinct design question; a wait node is not specified here. +Scheduling starts a new run of a saved deployment. Waiting until a time or +event during an existing run would instead require a durable suspension point +and rules for resuming it. Future work should distinguish these two forms of +timed execution rather than treat a wait operation as another schedule. +Distributed execution would additionally require an ownership model beyond +the current single-scheduler arrangement. + +## An Assistant-Backed Authoring Application The surrounding application is intended to combine assistant-backed chat with workflow administration. A shell can let an assistant retain Python objects across interactions, while typed client objects can support -dedicated views of artifacts, deployments, and runs. Specialized display -payloads, potentially using MIME types, are a presentation option to -investigate rather than an established public contract. +dedicated views of artifacts, deployments, and runs. The design question is +how to let an author inspect and correct a proposed procedure before executing +it, while keeping subsequent run status and requests for input understandable. -This direction does not establish a completed workflow-agent integration. -It needs a concrete interaction design, demonstrated public-client use, -and its own evaluation before contributing success claims to the thesis. +This application requires an interaction design and evaluation of how the +assistant uses the public client to construct, revise, and inspect workflows. -## Extend Operations When Concrete Use Requires Them +## Preserving Contracts Across Operational Changes -Provider expansion and operational hardening should similarly follow actual -requirements. Examples include broader OpenAPI coverage, source reload, -secret-manager integration, and an alternative storage backend. Each needs -its own compatibility, failure, and deployment evidence; none follows -automatically from the existence of a provider or store interface. +Additional providers and storage backends would test whether the architectural +boundaries hold beyond the demonstrated implementations. The question is +whether an integration can preserve source compatibility, run identity, and +recovery behavior without changing the graph's execution rules. Such work +needs failure tests and deployment evidence as well as a working adapter. Richer debugging should clarify what can safely be resumed or repeated, especially around external side effects. Showing more trace information is @@ -1551,19 +1710,19 @@ and interruption. The Python client exposes this model through authoring and inspection objects, while the API and provider boundaries connect it to configured operations. -The report case demonstrates that a small typed procedure can be saved, -deployed, executed, and inspected. Targeted tests support specific -validation, state, and persistence behaviors. These results support the -feasibility of the design under the tested conditions, -not a claim of production readiness or superior usability. +The report case demonstrates that these representations compose into a +save–deploy–run–inspect lifecycle for the fixed fixture. The repository also +contains targeted tests covering execution requirements X1–X5; the evidence +index locates them without reporting a combined execution result. +For the authoring objectives R1–R5, the +work identifies and exercises supporting interfaces, but their effectiveness +for independent human or agent authors remains an open evaluation question. -The central trade-off remains visible: explicit contracts and lifecycle -boundaries improve inspectability but ask authors to understand more than -a sequence of function calls. The next stage must therefore test both the -correctness of richer execution semantics and the clarity of the experience -used to author and diagnose them. A workflow system is useful only when its -execution rules are dependable and its users can understand what they have -asked it to do. +Explicit contracts expose data mappings, saved versions, and run records, +while requiring authors to learn more concepts than a sequence of function +calls. The implemented contribution is the integration of these contracts +into a programmable lifecycle, demonstrated by saving, configuring, executing, +and inspecting the report procedure through the public client. # References {#sec:refs .unnumbered} @@ -1599,6 +1758,7 @@ E2: reconstructed client objects, editable workflows, and the report fixture. - `tests/wf_client/test_runs.py` - `examples/report_workflow/` - `tests/examples/test_report_workflow_example.py` +- `tests/examples/test_thesis_python_walkthrough.py` The example's README retains the command-line route for operators who need it. That alternative interface is not an additional evaluated case in the diff --git a/examples/report_workflow/wf.config.json b/examples/report_workflow/wf.config.json index e7a9353d..cc65cd91 100644 --- a/examples/report_workflow/wf.config.json +++ b/examples/report_workflow/wf.config.json @@ -27,6 +27,13 @@ "path": ".", "module": "ops", "registry": "registry" + }, + { + "kind": "python", + "id": "local.report_runtime", + "path": ".", + "module": "ops", + "registry": "registry" } ] } diff --git a/tests/examples/test_thesis_python_walkthrough.py b/tests/examples/test_thesis_python_walkthrough.py new file mode 100644 index 00000000..44abfa60 --- /dev/null +++ b/tests/examples/test_thesis_python_walkthrough.py @@ -0,0 +1,76 @@ +"""Execute the thesis's Python session, substituting only its transport setup.""" + +from __future__ import annotations + +import ast +import re +from collections.abc import Coroutine +from pathlib import Path +from typing import Any, cast + +import pytest + +from wf_client import App, Deployment, Run +from wf_client.protocols import WorkflowClientPort +from wf_config import load_workflow_config +from wf_server.config import build_workflow_server_from_workflow_config + +ROOT = Path(__file__).resolve().parents[2] +THESIS = ROOT / "docs/thesis/system-design-implementation.md" + + +@pytest.mark.asyncio +async def test_thesis_python_session_preserves_report_and_repair( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Catch broken displayed bindings, client calls, and missing saved results. + + Execute trusted repository Markdown, not submitted workflow/user text. + No operations or persistence are mocked: only the displayed HTTP connection + is replaced by the real API in process. This does not test the HTTP adapter. + """ + config = load_workflow_config(ROOT / "examples/report_workflow/wf.config.json") + config.server.store.root = tmp_path / "store" + server = build_workflow_server_from_workflow_config(config) + app = App._from_port(cast(WorkflowClientPort, server.api)) + monkeypatch.chdir(ROOT) + + manuscript = THESIS.read_text(encoding="utf-8") + chapter = manuscript.split("# Case Study: Deterministic Report Workflow\n", 1)[1] + chapter = chapter.split("\n# Evaluation\n", 1)[0] + blocks = re.findall(r"^```python\n(.*?)^```", chapter, re.MULTILINE | re.DOTALL) + assert blocks, "The case study must contain an executable Python session" + session = "\n".join(blocks) + connection = 'app = App.from_http_jsonrpc("http://127.0.0.1:8771/rpc")' + assert session.count(connection) == 1, ( + "Review the documented transport substitution" + ) + session = session.replace(connection, "app = supplied_app") + namespace: dict[str, Any] = {"supplied_app": app} + # The session does not import future annotations; do not inherit this file's + # compiler flags and accidentally change Pydantic's model construction. + code = compile( + session, + str(THESIS), + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + dont_inherit=True, + ) + await cast(Coroutine[Any, Any, None], eval(code, namespace)) + + run = namespace["run"] + assert isinstance(run, Run) + assert run.status == "completed" + deployment = namespace["deployment"] + assert isinstance(deployment, Deployment) + assert deployment.bindings["local.report"] == "local.report_runtime" + stored = await server.api.inspect_run(run_id=run.run_id) + output = stored["output"] + assert output is not None + assert output["report"]["title"] == "Weekly Project Update" + assert len(output["report"]["action_items"]) == 3 + assert output["markdown"].startswith("# Weekly Project Update") + diagnostic = capsys.readouterr().out + assert "invalid_source_path" in diagnostic + assert "output[0].path" in diagnostic + assert namespace["trace"].frames