Add capture event transport and server-side write classification

.gitignore

@@ -21,5 +21,14 @@
 #
 # dist build output
 target/
+/server/bindings/
+
+# Runtime capture spools and generated capture exports.
+/capture-spool/
+/capture-metrics-*.csv
+/capture-analysis-*/
+/server/scripts/output
+/server/scripts/capture-metrics-*.csv
+/server/scripts/capture-analysis-*/
 
 # CodeChat Editor lexer: python. See TODO.

README.md

@@ -23,6 +23,42 @@ Install the
 [CodeChat Editor extension for Visual Studio code](extensions/VSCode/README.md).
 For developers, see [building from source](docs/design.md).
 
+Research capture
+----------------
+
+The VS Code extension can record dissertation study capture events when a
+participant explicitly opts in. A participant first registers in the capture
+portal, which emails a capture token. In VS Code, run **Manage CodeChat Editor
+Capture** or **CodeChat Editor: Enter Capture Token** from the command palette,
+paste the token, then turn on consent and recording from the same capture
+manager.
+
+The token is imported through the VS Code UI and persisted only in VS Code
+SecretStorage. It is never written to workspace settings, repository files, or a
+JSON configuration file. The extension asks CaptureWebService for token status;
+the status item and capture manager show whether the token is accepted,
+rejected, unavailable, or disabled by the portal. The participant ID used in
+events comes from that status response, not from the token text.
+
+CodeChat no longer connects directly to the remote capture database and no
+longer reads or stores database credentials. The old local JSON database-secret
+configuration path has been removed. Capture events now leave CodeChat only by
+calling CaptureWebService with the portal-issued bearer token; any database
+writer role remains inside the service deployment.
+
+Events are sanitized, written to a durable local FIFO spool, then uploaded to
+CaptureWebService. Spooled events carry only a non-secret token hash/service
+identity so events from an old token are not uploaded under a new token. Offline
+recording is allowed only after the same token and service URL have previously
+been verified as capture-enabled; a token disabled by the portal remains
+disabled while the service is unavailable. If the network or service is
+unavailable after the token has been accepted at least once, queued events remain
+in the spool and upload as soon as the matching token and service are available
+again. The capture service endpoint can be changed in the user-level
+`CodeChatEditor.Capture.ServiceBaseUrl` setting; workspace values are ignored
+for this token-bearing endpoint. Token-bearing requests require HTTPS except for
+localhost development endpoints.
+
 Structure
 ---------
 

client/src/CodeChatEditor-test.mts

@@ -45,6 +45,24 @@ import {
 // From [SO](https://stackoverflow.com/a/39914235).
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
 
+const RENDER_TIMEOUT_MS = 10000;
+const MOCHA_TEST_TIMEOUT_MS = RENDER_TIMEOUT_MS + 5000;
+
+const waitFor = async (
+    description: string,
+    predicate: () => boolean,
+    timeoutMs = RENDER_TIMEOUT_MS,
+) => {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+        if (predicate()) {
+            return;
+        }
+        await sleep(100);
+    }
+    assert.fail(`Timed out waiting for ${description}.`);
+};
+
 // Tests
 // -----
 //
@@ -97,25 +115,45 @@ window.CodeChatEditor_test = () => {
                 });
             });
 
-            test("GraphViz, Mathjax, Mermaid", async function () {
-                // Wait for the renderers to run.
-                await sleep(1500);
+            test("GraphViz, Mathjax, Mermaid", async function (this: Mocha.Context) {
+                this.timeout(MOCHA_TEST_TIMEOUT_MS);
+
                 // Make sure GraphViz includes a `div` at the top of the shadow
                 // root, with a `svg` inside it.
+                const getGraphVizRoot = () =>
+                    document.getElementsByTagName("graphviz-graph")[0]
+                        ?.shadowRoot?.children[0];
+                await waitFor(
+                    "GraphViz SVG",
+                    () => getGraphVizRoot()?.children[0]?.tagName === "svg",
+                );
                 const gv =
                     document.getElementsByTagName("graphviz-graph")[0]
                         .shadowRoot!.children[0];
                 assert.equal(gv.tagName, "DIV");
                 assert.equal(gv.children[0].tagName, "svg");
 
                 // Mermaid graphs start with a div.
+                const getMermaidRoot = () =>
+                    document.getElementsByTagName("wc-mermaid")[0]?.shadowRoot
+                        ?.children[0];
+                await waitFor(
+                    "Mermaid SVG",
+                    () => getMermaidRoot()?.children[0]?.tagName === "svg",
+                );
                 const mer =
                     document.getElementsByTagName("wc-mermaid")[0].shadowRoot!
                         .children[0];
                 assert.equal(mer.tagName, "DIV");
                 assert.equal(mer.children[0].tagName, "svg");
 
                 // MathJax has its own stuff.
+                await waitFor(
+                    "MathJax containers",
+                    () =>
+                        document.getElementsByTagName("mjx-container")
+                            .length === 2,
+                );
                 assert.equal(
                     document.getElementsByTagName("mjx-container").length,
                     2,

client/src/CodeChatEditor.mts

@@ -716,6 +716,9 @@ export const on_error = (event: Event) => {
     let err_str: string;
     if (event instanceof ErrorEvent) {
         err_str = `${event.filename}:${event.lineno}: ${event.message}`;
+        if (event.error?.stack) {
+            err_str += `\n${event.error.stack}`;
+        }
     } else if (event instanceof PromiseRejectionEvent) {
         const reason = event.reason;
         let userMessage = "An unexpected error occurred. Please try again.";

client/src/shared.mts

@@ -43,6 +43,8 @@ import { CodeMirrorDocBlockTuple } from "./rust-types/CodeMirrorDocBlockTuple.js
 import { UpdateMessageContents } from "./rust-types/UpdateMessageContents.js";
 import { ResultOkTypes } from "./rust-types/ResultOkTypes.js";
 import { ResultErrTypes } from "./rust-types/ResultErrTypes.js";
+import { CaptureEventWire } from "./rust-types/CaptureEventWire.js";
+import { CaptureStatus } from "./rust-types/CaptureStatus.js";
 
 // Manually define this, since `ts-rs` can't export `webserver.MessageResult`.
 type MessageResult = { Ok: ResultOkTypes } | { Err: ResultErrTypes };
@@ -55,6 +57,8 @@ export type {
     CodeMirror,
     CodeMirrorDiffable,
     CodeMirrorDocBlockTuple,
+    CaptureEventWire,
+    CaptureStatus,
     CodeChatForWeb,
     EditorMessage,
     EditorMessageContents,

docs/implementation.md

@@ -60,9 +60,46 @@ Inside the client:
 The entire VSCode interface is contained in the extension, with the NAPI-RS glue
 in the corresponding library.
 
-Does this make more sense to place in the TOC? Or is it too wordy there? I think
-a diagram as an overview might be helpful. Perhaps the server, client, etc.
-should have its of readme files providing some of this.
+### Capture path
+
+Dissertation capture is a web-service-only path. The VS Code extension stores
+the portal-issued capture token in VS Code SecretStorage after the user enters
+it through the capture manager or **CodeChat Editor: Enter Capture Token**
+command. The token is never written to settings, workspace files, or a JSON
+configuration file. The extension keeps only non-secret participant/instance
+metadata in VS Code global state and asks CaptureWebService for token status
+before recording user events. The participant ID in each event comes from that
+status response. The extension also caches the last capture-enabled decision for
+the same token hash and service URL, so offline fallback cannot turn a
+portal-disabled token back into a recordable state.
+
+When consent, recording, and token status allow capture, the extension sends the
+same capture event shape to the local CodeChat server through the existing
+NAPI-RS bridge. The server hashes raw local file paths, removes any raw path
+fields from event data before local persistence, writes every event to a durable
+`capture-spool` FIFO directory, and uploads batches to
+`POST /v1/capture/events` using
+`Authorization: Bearer <token>`. CodeChat does not read database credentials
+from disk and does not connect directly to PostgreSQL; the database schema under
+`server/scripts/` documents the server-side table used by CaptureWebService.
+The previous local JSON database-secret configuration path is intentionally
+removed; any database writer role or password must remain server-side in the
+CaptureWebService deployment.
+The service endpoint is read only from the user/application-level
+`CodeChatEditor.Capture.ServiceBaseUrl` setting; workspace values are ignored so
+a repository cannot redirect a stored token. Bearer-token requests require
+HTTPS, with `http://localhost` and `http://127.0.0.1` allowed only for local
+development.
+
+The upload worker deletes spooled events only after the service returns `202`.
+After a token has been accepted at least once, transient service or network
+failures still allow local recording and keep events queued for retry. Each
+spool record includes a non-secret token hash and service URL identity; the
+worker uploads only records matching the currently configured token/service, so
+events queued under an old token remain local until that matching token is
+configured again. Authentication failures pause upload until a valid token is
+entered, malformed or oversized events are moved to the spool quarantine
+directory, and blocking service calls use bounded request timeouts.
 
 <a id="an-implementation"></a>Architecture
 ------------------------------------------

extensions/VSCode/.gitignore

@@ -22,6 +22,7 @@
 # NPM
 node_modules/
 out/
+.test-output/
 
 # Server support files
 log4rs.yml
@@ -33,5 +34,5 @@ src/index.d.ts
 src/index.js
 src/codechat-editor-client.win32-x64-msvc.node
 .windows/
-
+*.log
 # CodeChat Editor lexer: python. See TODO.

extensions/VSCode/.vscodeignore

@@ -27,6 +27,7 @@ src/**
 
 # Omit libraries, since `esbuild` takes care of packaging.
 node_modules/
+.test-output/**
 
 # Omit VSCode config.
 .vscode/**
@@ -41,6 +42,7 @@ target/**
 # Misc files not needed in a package.
 .eslintrc.yml
 .gitignore
+out/*.test.cjs
 build.rs
 Cargo.lock
 Cargo.toml