docs/CONTRIBUTING_SQL.md
This guide covers how to add new SQL features, functions, and PostgreSQL compatibility improvements to immudb.
Client (psql, pgx, JDBC, ORM)
|
v
pkg/pgsql/server/ -- PostgreSQL v3 wire protocol
|
v
embedded/sql/engine.go -- SQL engine (parse, plan, execute)
embedded/sql/stmt.go -- AST nodes and statement execution
embedded/sql/sql_grammar.y -- Yacc grammar (generates sql_parser.go)
embedded/sql/parser.go -- Lexer and keyword mapping
embedded/sql/catalog.go -- Schema metadata (tables, columns, indexes)
embedded/sql/functions.go -- Built-in SQL functions
|
v
embedded/store/ -- Immutable key-value store with Merkle trees
// SQLStmt — any executable SQL statement
type SQLStmt interface {
readOnly() bool
requiredPrivileges() []SQLPrivilege
execAt(ctx, tx, params) (*SQLTx, error)
inferParameters(ctx, tx, params)
}
// DataSource — a statement that returns rows (SELECT, UNION, CTE, RETURNING)
type DataSource interface {
SQLStmt
Resolve(ctx, tx, params, scanSpecs) (RowReader, error)
Alias() string
}
// Function — a built-in SQL function
type Function interface {
InferType(cols, params, implicitTable) (SQLValueType, error)
RequiresType(t, cols, params, implicitTable) error
Apply(tx *SQLTx, params []TypedValue) (TypedValue, error)
}
// TableResolver — virtual table (pg_catalog, views, CTEs)
type TableResolver interface {
Table() string
Resolve(ctx, tx, alias) (RowReader, error)
}
In embedded/sql/functions.go:
const (
// ... existing constants ...
MyNewFnCall string = "MY_NEW_FN"
)
type myNewFn struct{}
func (f *myNewFn) InferType(cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) (SQLValueType, error) {
return VarcharType, nil // return type
}
func (f *myNewFn) RequiresType(t SQLValueType, cols map[string]ColDescriptor, params map[string]SQLValueType, implicitTable string) error {
if t != VarcharType {
return fmt.Errorf("%w: %v can not be interpreted as type %v", ErrInvalidTypes, VarcharType, t)
}
return nil
}
func (f *myNewFn) Apply(tx *SQLTx, params []TypedValue) (TypedValue, error) {
if len(params) != 1 {
return nil, fmt.Errorf("%w: '%s' expects 1 argument", ErrIllegalArguments, MyNewFnCall)
}
if params[0].IsNull() {
return NewNull(VarcharType), nil
}
// ... implement logic ...
return NewVarchar(result), nil
}
var builtinFunctions = map[string]Function{
// ... existing entries ...
MyNewFnCall: &myNewFn{},
}
See Testing Requirements below.
pgTableIsVisible, pgCurrentSchemaCoalesceFn, ConcatFnNowFn (timestamp), pgGetUserByIDFunc (users)mathFn (generic unary/binary math)params[0].IsNull() and return NewNull(type)In embedded/sql/parser.go, add to the keywords map:
var keywords = map[string]int{
// ... existing keywords ...
"MYNEWKW": MYNEWKW,
}
In embedded/sql/sql_grammar.y:
%token <keyword> ... MYNEWKW
my_new_stmt:
MYNEWKW IDENTIFIER
{
$$ = &MyNewStmt{name: $2}
}
Add the rule to the appropriate parent rule (ddlstmt, dmlstmt, or dqlstmt).
In embedded/sql/stmt.go:
type MyNewStmt struct {
name string
}
func (stmt *MyNewStmt) readOnly() bool { return true }
func (stmt *MyNewStmt) requiredPrivileges() []SQLPrivilege { return nil }
func (stmt *MyNewStmt) inferParameters(ctx context.Context, tx *SQLTx, params map[string]SQLValueType) error {
return nil
}
func (stmt *MyNewStmt) execAt(ctx context.Context, tx *SQLTx, params map[string]interface{}) (*SQLTx, error) {
// ... implement execution ...
return tx, nil
}
go run golang.org/x/tools/cmd/goyacc -l -o embedded/sql/sql_parser.go embedded/sql/sql_grammar.y
One shift/reduce conflict is expected (from CROSS JOIN optional ON clause). Additional conflicts should be investigated.
See Testing Requirements below.
In pkg/pgsql/pgschema/table_resolvers.go:
var myTableCols = []sql.ColDescriptor{
{Column: "col1", Type: sql.IntegerType},
{Column: "col2", Type: sql.VarcharType},
}
type myTableResolver struct{}
func (r *myTableResolver) Table() string { return "my_table" }
func (r *myTableResolver) Resolve(ctx context.Context, tx *sql.SQLTx, alias string) (sql.RowReader, error) {
catalog := tx.Catalog()
tables := catalog.GetTables()
var rows [][]sql.ValueExp
for _, t := range tables {
rows = append(rows, []sql.ValueExp{
sql.NewInteger(int64(t.ID())),
sql.NewVarchar(t.Name()),
})
}
return sql.NewValuesRowReader(tx, nil, myTableCols, true, alias, rows)
}
Add to the tableResolvers slice at the bottom of the file:
var tableResolvers = []sql.TableResolver{
// ... existing resolvers ...
&myTableResolver{},
}
Add a test in pkg/pgsql/pgschema/resolvers_test.go following the existing pattern.
In pkg/pgsql/server/immudb_functions.go, add to the showSettings map:
var showSettings = map[string]string{
// ... existing settings ...
"my_setting": "my_value",
}
In pkg/pgsql/server/stmts_handler.go, add a regex pattern and handler:
var myFuncRe = regexp.MustCompile(`(?i)select\s+my_func\(\s*\)`)
Add to isEmulableInternally() and tryToHandleInternally().
Every new feature must have tests at multiple levels:
In embedded/sql/engine_test.go:
func TestMyNewFeature(t *testing.T) {
engine := setupCommonTest(t)
_, _, err := engine.Exec(context.Background(), nil,
`CREATE TABLE test (id INTEGER, PRIMARY KEY id)`, nil)
require.NoError(t, err)
r, err := engine.Query(context.Background(), nil,
`SELECT MY_NEW_FN('input')`, nil)
require.NoError(t, err)
row, err := r.Read(context.Background())
require.NoError(t, err)
require.Equal(t, expectedValue, row.ValuesByPosition[0].RawValue())
r.Close()
}
In embedded/sql/new_features_edge_test.go:
In pkg/pgsql/server/pgsql_compat_integration_test.go:
func TestPgsqlCompat_MyFeature(t *testing.T) {
_, port := setupTestServer(t)
conn, err := pgx.Connect(context.Background(),
fmt.Sprintf("host=localhost port=%d sslmode=disable user=immudb dbname=defaultdb password=immudb", port))
require.NoError(t, err)
defer conn.Close(context.Background())
// Test through actual PG wire protocol
rows, err := conn.Query(context.Background(), "SELECT MY_NEW_FN('test')")
require.NoError(t, err)
require.True(t, rows.Next())
rows.Close()
}
In embedded/sql/new_features_stress_test.go:
setupCommonTest(t) for engine testssetupTestServer(t) for PG wire tests — returns (server, port)r.Close() or rows.Close() to avoid snapshot leaksrequire.ErrorIs(t, err, ErrNoMoreRows) to check for end of resultstime.Sleep in tests — immudb's async index makes timing-dependent tests flakyimmudb is append-only. All mutations create new versions, never modify or delete physical data:
DELETE marks rows as deleted (soft delete)UPDATE creates a new version of the rowtx.doUpsert or tx.deleteIndexEntriesimmudb's LIKE and ILIKE operators use Go regex syntax, not SQL %/_ wildcards:
'hello.*' not 'hello%''.' not '_'New metadata that must survive restarts needs catalog persistence:
stmt.go (e.g., catalogSequencePrefix = "CTL.SEQUENCE.")persist* function to write to KV storecatalog.go or engine.go (NewTx path){flags:1byte}{maxLen:4bytes}{colName}Features that are session-scoped (lost on restart):
The PG wire handler dispatches by type:
sql.DataSource implementations → query() path (returns rows)exec() path (no rows returned)RETURNING makes DML statements implement DataSourceThe grammar has 1 expected shift/reduce conflict from the CROSS JOIN optional ON clause. Yacc resolves it correctly (shift prefers ON when present). Additional conflicts should be investigated.
# SQL engine tests
go test ./embedded/sql/ -count=1 -short -timeout 300s
# PG wire protocol tests
go test ./pkg/pgsql/... -count=1 -short -timeout 120s
# pg_catalog/information_schema resolver tests
go test ./pkg/pgsql/pgschema/ -count=1 -v
# Full test suite (all packages)
go test ./embedded/... ./pkg/... -count=1 -short -timeout 600s
# Specific feature tests
go test ./embedded/sql/ -run "TestWindowFunctions" -count=1 -v
# PG wire integration tests only
go test ./pkg/pgsql/server/ -run "TestPgsqlCompat_" -count=1 -v
| File | Purpose |
|---|---|
embedded/sql/sql_grammar.y | Yacc grammar — all SQL syntax rules |
embedded/sql/sql_parser.go | Generated parser (do not edit manually) |
embedded/sql/parser.go | Lexer, keyword map, token handling |
embedded/sql/stmt.go | AST nodes, statement types, execution logic |
embedded/sql/engine.go | SQL engine, transaction management, sequences |
embedded/sql/catalog.go | Schema metadata (tables, columns, indexes) |
embedded/sql/functions.go | All built-in SQL functions |
embedded/sql/row_reader.go | Base row reader, raw KV scan |
embedded/sql/cond_row_reader.go | WHERE clause filtering |
embedded/sql/joint_row_reader.go | JOIN execution (INNER, LEFT, CROSS) |
embedded/sql/full_outer_join_reader.go | FULL OUTER JOIN (materializing) |
embedded/sql/sort_reader.go | ORDER BY sorting |
embedded/sql/grouped_row_reader.go | GROUP BY aggregation |
embedded/sql/distinct_row_reader.go | DISTINCT filtering |
embedded/sql/union_row_reader.go | UNION execution |
embedded/sql/set_op_row_reader.go | EXCEPT/INTERSECT execution |
embedded/sql/window_row_reader.go | Window function execution |
embedded/sql/proj_row_reader.go | Column projection and aliases |
embedded/sql/limit_row_reader.go | LIMIT clause |
embedded/sql/offset_row_reader.go | OFFSET clause |
embedded/sql/values_row_reader.go | Literal VALUES and virtual tables |
pkg/pgsql/server/query_machine.go | PG wire protocol query dispatch |
pkg/pgsql/server/stmts_handler.go | Statement classification and emulation |
pkg/pgsql/server/immudb_functions.go | immudb verification SQL functions |
pkg/pgsql/server/session.go | PG wire session management |
pkg/pgsql/server/bmessages/ | PG wire backend messages (responses) |
pkg/pgsql/server/fmessages/ | PG wire frontend messages (requests) |
pkg/pgsql/server/pgmeta/pg_type.go | PG type OID mapping |
pkg/pgsql/pgschema/table_resolvers.go | pg_catalog and information_schema resolvers |