feat(kernel): SEA-via-kernel backend (opt-in, behind build tag)

connector.go

@@ -15,6 +15,7 @@ import (
 	"github.com/databricks/databricks-sql-go/auth/pat"
 	"github.com/databricks/databricks-sql-go/auth/tokenprovider"
 	"github.com/databricks/databricks-sql-go/driverctx"
+	"github.com/databricks/databricks-sql-go/internal/backend"
 	"github.com/databricks/databricks-sql-go/internal/backend/thrift"
 	"github.com/databricks/databricks-sql-go/internal/client"
 	"github.com/databricks/databricks-sql-go/internal/config"
@@ -32,9 +33,18 @@ type connector struct {
 func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	defer debuglog.Track(ctx, "connector.Connect", "host=%s", c.cfg.Host)()
 
-	// Build the execution backend. Currently always the Thrift backend; a second
-	// SEA-via-kernel backend will be selected here by a connect-time flag.
-	be, err := thrift.New(ctx, c.cfg, c.client)
+	// Build the execution backend. Thrift is the default; the SEA-via-kernel
+	// backend is selected when UseKernel is set. newKernelBackend is build-tag
+	// gated: in the default pure-Go build it returns a clear "not linked in"
+	// error, so the kernel path compiles and links only under -tags
+	// databricks_kernel + CGO_ENABLED=1.
+	var be backend.Backend
+	var err error
+	if c.cfg.UseKernel {
+		be, err = newKernelBackend(ctx, c.cfg)
+	} else {
+		be, err = thrift.New(ctx, c.cfg, c.client)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -77,7 +87,14 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 		conn.telemetry.RecordOperation(ctx, conn.id, "", telemetry.OperationTypeCreateSession, sessionLatencyMs, nil)
 	}
 
-	log.Info().Msgf("connect: host=%s port=%d httpPath=%s serverProtocolVersion=0x%X", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath, be.ServerProtocolVersion())
+	// ServerProtocolVersion is Thrift-specific (not on the neutral backend
+	// interface); the kernel backend has no negotiated Thrift protocol, so log it
+	// only when present.
+	if tb, ok := be.(*thrift.Backend); ok {
+		log.Info().Msgf("connect: host=%s port=%d httpPath=%s serverProtocolVersion=0x%X", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath, tb.ServerProtocolVersion())
+	} else {
+		log.Info().Msgf("connect: host=%s port=%d httpPath=%s backend=kernel", c.cfg.Host, c.cfg.Port, c.cfg.HTTPPath)
+	}
 
 	return conn, nil
 }
@@ -301,6 +318,26 @@ func WithHTTPPath(path string) ConnOption {
 	}
 }
 
+// WithUseKernel selects the SEA-via-kernel backend instead of the default
+// Thrift backend. It has effect only in a build compiled with
+// `-tags databricks_kernel` and CGO_ENABLED=1; in the default pure-Go build a
+// connection made with this option set returns a clear error at connect time
+// (the kernel backend is not linked in).
+func WithUseKernel(useKernel bool) ConnOption {
+	return func(c *config.Config) {
+		c.UseKernel = useKernel
+	}
+}
+
+// WithWarehouseID sets the bare SQL warehouse id. The kernel backend addresses a
+// warehouse by id; when set it is preferred over the http path. The Thrift
+// backend ignores it and continues to route by http path.
+func WithWarehouseID(id string) ConnOption {
+	return func(c *config.Config) {
+		c.WarehouseID = id
+	}
+}
+
 // WithMaxRows sets up the max rows fetched per request. Default is 10000
 func WithMaxRows(n int) ConnOption {
 	return func(c *config.Config) {

doc.go

@@ -155,6 +155,42 @@ The result log may look like this:
 
 	{"level":"debug","connId":"01ed6545-5669-1ec7-8c7e-6d8a1ea0ab16","corrId":"workflow-example","queryId":"01ed6545-57cc-188a-bfc5-d9c0eaf8e189","time":1668558402,"message":"Run Main elapsed time: 1.298712292s"}
 
+# SEA-via-kernel backend (experimental)
+
+By default the driver uses the Thrift/HiveServer2 backend and is pure Go
+(CGO_ENABLED=0, go-gettable, cross-compilable). An experimental second backend
+runs statements over the Statement Execution API via the Rust
+databricks-sql-kernel, reached through a cgo C ABI. It is opt-in and compiled in
+only under a build tag, so the default build is unchanged.
+
+To use it, build with the databricks_kernel tag and CGO enabled, and select it
+per connection with WithUseKernel:
+
+	CGO_ENABLED=1 go build -tags databricks_kernel ./...
+
+	connector, _ := dbsql.NewConnector(
+		dbsql.WithServerHostname(host),
+		dbsql.WithHTTPPath(httpPath),
+		dbsql.WithAccessToken(token),
+		dbsql.WithUseKernel(true),
+	)
+	db := sql.OpenDB(connector)
+
+In a build without the tag, WithUseKernel(true) returns a clear error at connect
+time rather than silently using Thrift.
+
+To see the kernel's own logs interleaved with the driver's, enable both on the
+same stderr. The kernel filters on the target databricks::sql::kernel (note the
+colons — it is not the crate module path), and the driver's step tracer is
+enabled by the standard logging level:
+
+	RUST_LOG=databricks::sql::kernel=debug DATABRICKS_LOG_LEVEL=debug ./your_app 2>&1
+
+The kernel backend currently supports PAT authentication and reading
+scalar-typed results (CloudFetch is handled transparently), with context
+cancellation. Bound parameters, nested and complex types, and other
+authentication and transport options are not yet supported on this backend.
+
 # Programmatically Retrieving Connection and Query Id
 
 Use the driverctx package under driverctx/ctx.go to add callbacks to the query context to receive the connection id and query id.

internal/backend/kernel/backend.go

@@ -0,0 +1,193 @@
+//go:build cgo && databricks_kernel
+
+package kernel
+
+/*
+#include <stdlib.h>
+#include "databricks_kernel.h"
+*/
+import "C"
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/databricks/databricks-sql-go/internal/backend"
+)
+
+// Config is the flat connection config for the kernel backend. The connector
+// fills it from the driver's config so the user-facing options are unchanged
+// (this mirrors how the kernel's pyo3/napi bindings take flat connection
+// params). Zero-valued fields are simply not applied.
+type Config struct {
+	Host        string // workspace hostname, no scheme
+	HTTPPath    string // e.g. /sql/1.0/warehouses/abc123 (carries ?o= org routing)
+	WarehouseID string // bare warehouse id; preferred over HTTPPath when set
+	Token       string // PAT (dapi...)
+
+	// SessionConf carries server-bound session confs verbatim — the same map the
+	// Thrift backend forwards (STATEMENT_TIMEOUT, QUERY_TAGS, TIMEZONE, …).
+	SessionConf map[string]string
+
+	// TLSSkipVerify disables server-cert hostname verification (maps the driver's
+	// WithSkipTLSHostVerify / TLSConfig.InsecureSkipVerify).
+	TLSSkipVerify bool
+
+	// ProxyURL configures an HTTP proxy, already resolved for this endpoint from
+	// the same HTTP(S)_PROXY / NO_PROXY environment the Thrift path uses (NO_PROXY
+	// is applied during resolution). Empty leaves the kernel on a direct
+	// connection.
+	ProxyURL string
+
+	// Location is the session time zone used to render DATE / TIMESTAMP values,
+	// matching the Thrift path which returns them in this location. nil means UTC.
+	Location *time.Location
+}
+
+// KernelBackend implements backend.Backend over the kernel C ABI. One backend
+// backs one conn, which database/sql serializes to a single goroutine at a time,
+// so the kernel session inherits single-owner-ship and needs no locks; the only
+// concurrency is the per-statement cancel watcher (see operation.go), which
+// touches only the kernel's internal inflight-id slot.
+type KernelBackend struct {
+	cfg       Config
+	session   *C.kernel_session_t
+	sessionID string
+	valid     bool
+}
+
+var _ backend.Backend = (*KernelBackend)(nil)
+
+// New builds a kernel backend without opening the session; the connector calls
+// OpenSession immediately after, mirroring the Thrift backend's shape.
+func New(cfg Config) *KernelBackend {
+	return &KernelBackend{cfg: cfg}
+}
+
+// OpenSession builds a session config (warehouse/http-path + PAT), opens the
+// session, and captures a per-conn id. Called once by the connector at connect
+// time. The config handle is consumed by kernel_session_open on success and
+// freed by us on any earlier failure.
+func (k *KernelBackend) OpenSession(ctx context.Context) error {
+	klog("OpenSession host=%s httpPath=%s warehouse=%s", k.cfg.Host, k.cfg.HTTPPath, k.cfg.WarehouseID)
+
+	var cfg *C.KernelSessionConfig
+	if err := call(func() C.KernelStatusCode { return C.kernel_session_config_new(&cfg) }); err != nil {
+		return fmt.Errorf("kernel: config_new: %w", toDriverError(err))
+	}
+	consumed := false
+	defer func() {
+		if !consumed {
+			C.kernel_session_config_free(cfg)
+		}
+	}()
+
+	// Warehouse addressing: bare id when provided, else the http path (which also
+	// carries ?o= org routing for shared hosts).
+	host := newCStr(k.cfg.Host)
+	defer host.free()
+	if k.cfg.WarehouseID != "" {
+		wh := newCStr(k.cfg.WarehouseID)
+		defer wh.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_warehouse(cfg, host.c, wh.c)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_warehouse: %w", toDriverError(err))
+		}
+	} else {
+		path := newCStr(k.cfg.HTTPPath)
+		defer path.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_http_path(cfg, host.c, path.c)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_http_path: %w", toDriverError(err))
+		}
+	}
+
+	tok := newCStr(k.cfg.Token)
+	defer tok.free()
+	if err := call(func() C.KernelStatusCode {
+		return C.kernel_session_config_set_auth_pat(cfg, tok.c)
+	}); err != nil {
+		return fmt.Errorf("kernel: set_auth_pat: %w", toDriverError(err))
+	}
+
+	// TLS: skip server-cert hostname verification when the driver requested it.
+	if k.cfg.TLSSkipVerify {
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_tls_skip_hostname_verification(cfg, C.bool(true))
+		}); err != nil {
+			return fmt.Errorf("kernel: set_tls_skip_hostname_verification: %w", toDriverError(err))
+		}
+	}
+
+	// Proxy: only when the environment configured one for this endpoint. NO_PROXY
+	// was already applied during resolution, so no bypass list is needed here;
+	// any credentials are carried in the URL userinfo (Go's proxy-env convention),
+	// so username/password are NULL.
+	if k.cfg.ProxyURL != "" {
+		url := newCStr(k.cfg.ProxyURL)
+		defer url.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_proxy(cfg, url.c, nil, nil, nil)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_proxy: %w", toDriverError(err))
+		}
+	}
+
+	// Session confs (STATEMENT_TIMEOUT, QUERY_TAGS, TIMEZONE, …) — the same map
+	// the Thrift backend forwards, applied one key at a time.
+	for key, val := range k.cfg.SessionConf {
+		ck := newCStr(key)
+		cv := newCStr(val)
+		errSet := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_session_conf(cfg, ck.c, cv.c)
+		})
+		ck.free()
+		cv.free()
+		if errSet != nil {
+			return fmt.Errorf("kernel: set_session_conf[%s]: %w", key, toDriverError(errSet))
+		}
+	}
+
+	var sess *C.kernel_session_t
+	if err := call(func() C.KernelStatusCode { return C.kernel_session_open(cfg, &sess) }); err != nil {
+		return fmt.Errorf("kernel: session_open: %w", toDriverError(err))
+	}
+	consumed = true // kernel_session_open took ownership of cfg
+	k.session = sess
+	k.valid = true
+	// The C ABI exposes no formatted session-id accessor; use the handle pointer
+	// as a stable per-conn id for logging / telemetry correlation.
+	k.sessionID = fmt.Sprintf("kernel-%p", sess)
+	klog("OpenSession OK session=%s", k.sessionID)
+	return nil
+}
+
+// CloseSession tears down the server-side session. Best-effort: the kernel's
+// close is async (see the C header), so an error is logged, not hard-failed.
+func (k *KernelBackend) CloseSession(ctx context.Context) error {
+	if k.session == nil {
+		return nil
+	}
+	klog("CloseSession session=%s", k.sessionID)
+	err := call(func() C.KernelStatusCode { return C.kernel_session_close(k.session) })
+	k.session = nil
+	k.valid = false
+	return toDriverError(err)
+}
+
+// SessionValid backs conn.IsValid → pool eviction. No I/O; inspects state
+// captured at OpenSession.
+func (k *KernelBackend) SessionValid() bool { return k.valid && k.session != nil }
+
+// SessionID is the per-conn id (conn.id). Valid after OpenSession.
+func (k *KernelBackend) SessionID() string { return k.sessionID }
+
+// Execute runs a statement to a terminal state via the blocking execute path.
+// Per the Backend contract it returns a non-nil Operation even on error so the
+// caller can read StatementID / wrap the error / Close uniformly.
+func (k *KernelBackend) Execute(ctx context.Context, req backend.ExecRequest) (backend.Operation, error) {
+	return k.execute(ctx, req)
+}