| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728 |
- package storage
- import (
- "errors"
- "fmt"
- "strings"
- "sync"
- "time"
- "github.com/goccy/go-json"
- "github.com/danfragoso/pizzasql-next/pkg/analyzer"
- )
- // ErrIndexNotFound is returned by GetIndex when an index is absent. It is
- // distinct from a storage/IO error so callers like ListTableIndexes can skip a
- // concurrently-dropped index without swallowing real read failures.
- var ErrIndexNotFound = errors.New("index not found")
- // Schema represents a table schema.
- type Schema struct {
- Name string `json:"name"`
- Columns []Column `json:"columns"`
- PrimaryKey string `json:"primary_key"`
- CreatedAt time.Time `json:"created_at"`
- NextRowID int64 `json:"next_rowid"`
- AutoIncrement bool `json:"autoincrement"`
- // Checks holds the row predicates declared with CHECK, at column and table
- // level alike. Inline and table-level declarations are equivalent once
- // parsed, so both live in this one list.
- Checks []CheckConstraint `json:"checks,omitempty"`
- }
- // CheckConstraint is a persisted CHECK predicate. The predicate is stored as
- // SQL text and re-parsed by the evaluator installed on the SchemaManager, so a
- // durable reload keeps enforcing it.
- type CheckConstraint struct {
- Name string `json:"name,omitempty"`
- Expression string `json:"expression"`
- }
- // Label identifies the constraint in error messages and the catalog: the
- // declared name when there is one, otherwise the expression text.
- func (c CheckConstraint) Label() string {
- if c.Name != "" {
- return c.Name
- }
- return c.Expression
- }
- // Column represents a column definition.
- type Column struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Nullable bool `json:"nullable"`
- Default interface{} `json:"default,omitempty"`
- PrimaryKey bool `json:"primary_key"`
- // GeneratedExpr holds the SQL text of a GENERATED ALWAYS AS (expr) column.
- // GeneratedStored reports whether the value is materialized on write
- // (STORED, the only form this engine persists) versus computed on read.
- GeneratedExpr string `json:"generated_expr,omitempty"`
- GeneratedStored bool `json:"generated_stored,omitempty"`
- }
- // Index represents an index definition.
- type Index struct {
- Name string `json:"name"`
- Table string `json:"table"`
- Columns []IndexColumn `json:"columns"`
- Unique bool `json:"unique"`
- CreatedAt time.Time `json:"created_at"`
- // OnConflict is the default conflict resolution declared for this index via
- // a UNIQUE(...) ON CONFLICT clause. Empty means the SQLite default (ABORT).
- OnConflict string `json:"on_conflict,omitempty"`
- }
- // IndexColumn represents a column in an index. Expression is set for expression
- // indexes (e.g. lower(email)); a plain column index leaves it empty and uses
- // Name.
- type IndexColumn struct {
- Name string `json:"name"`
- Desc bool `json:"desc"`
- Expression string `json:"expression,omitempty"`
- }
- // rowIDAllocator owns the next-ROWID state for a single table. It is a separate
- // mutex per table so allocating a ROWID on one table never serializes against
- // another table, and never contends with the SchemaManager catalog lock.
- type rowIDAllocator struct {
- mu sync.Mutex
- next int64
- init bool
- }
- // SchemaManager manages table schemas.
- type SchemaManager struct {
- pool *KVPool
- database string
- cache map[string]*Schema
- indexCache map[string]*Index
- indexListCache []string
- indexListCached bool
- version uint64
- mu sync.RWMutex
- tableLocksMu sync.Mutex
- tableLocks map[string]*sync.RWMutex
- // checkMu guards checkEval, the pluggable evaluator for persisted CHECK
- // predicates. It is set by the SQL executor so storage stays independent of
- // the SQL front end.
- checkMu sync.RWMutex
- checkEval CheckEvaluator
- rowIDMu sync.Mutex
- rowIDAlloc map[string]*rowIDAllocator
- }
- // CheckEvaluator reports whether a persisted CHECK expression holds for a row.
- // A false result is a violation; NULL and other unknown results must be
- // reported as true, matching SQL three-valued logic. The SQL executor registers
- // one so the storage layer can enforce CHECK without importing the parser.
- type CheckEvaluator func(expression string, row Row) (bool, error)
- // SetCheckEvaluator installs the evaluator used to enforce persisted CHECK
- // predicates.
- func (m *SchemaManager) SetCheckEvaluator(eval CheckEvaluator) {
- m.checkMu.Lock()
- m.checkEval = eval
- m.checkMu.Unlock()
- }
- func (m *SchemaManager) checkEvaluator() CheckEvaluator {
- m.checkMu.RLock()
- defer m.checkMu.RUnlock()
- return m.checkEval
- }
- // BeginTransaction is retained for API compatibility. Buffered per-session
- // transactions no longer take a database-wide transaction lock; staged writes
- // are validated and committed atomically with CompareBatchWrite instead.
- func (m *SchemaManager) BeginTransaction() {}
- // EndTransaction is retained for API compatibility.
- func (m *SchemaManager) EndTransaction() {}
- // LockStatement is retained for API compatibility. Statement execution is now
- // serialized through per-table locks and optimistic validation, so no global
- // statement lock is required.
- func (m *SchemaManager) LockStatement() {}
- // UnlockStatement is retained for API compatibility.
- func (m *SchemaManager) UnlockStatement() {}
- // NewSchemaManager creates a new schema manager.
- func NewSchemaManager(pool *KVPool, database string) *SchemaManager {
- return &SchemaManager{
- pool: pool,
- database: database,
- cache: make(map[string]*Schema),
- indexCache: make(map[string]*Index),
- tableLocks: make(map[string]*sync.RWMutex),
- rowIDAlloc: make(map[string]*rowIDAllocator),
- }
- }
- // rowIDAllocatorFor returns (creating if needed) the per-table ROWID allocator.
- func (m *SchemaManager) rowIDAllocatorFor(table string) *rowIDAllocator {
- key := strings.ToLower(table)
- m.rowIDMu.Lock()
- a, ok := m.rowIDAlloc[key]
- if !ok {
- a = &rowIDAllocator{}
- m.rowIDAlloc[key] = a
- }
- m.rowIDMu.Unlock()
- return a
- }
- func (m *SchemaManager) tableLock(table string) *sync.RWMutex {
- key := strings.ToLower(table)
- m.tableLocksMu.Lock()
- lock, ok := m.tableLocks[key]
- if !ok {
- lock = &sync.RWMutex{}
- m.tableLocks[key] = lock
- }
- m.tableLocksMu.Unlock()
- return lock
- }
- // GetDatabaseName returns the database name.
- func (m *SchemaManager) GetDatabaseName() string {
- return m.database
- }
- // GetPool returns the KV pool.
- func (m *SchemaManager) GetPool() *KVPool {
- return m.pool
- }
- // Version returns the in-process schema catalog version. It is incremented for
- // schema/index definition changes so cached executors can resync their analyzer
- // catalogs without scanning storage on every query.
- func (m *SchemaManager) Version() uint64 {
- m.mu.RLock()
- defer m.mu.RUnlock()
- return m.version
- }
- func (m *SchemaManager) bumpVersionLocked() {
- m.version++
- }
- // schemaKey returns the key for a table schema.
- func (m *SchemaManager) schemaKey(table string) string {
- return fmt.Sprintf("%s:_schema:%s", m.database, strings.ToLower(table))
- }
- // catalogKey returns the key for the table catalog.
- func (m *SchemaManager) catalogKey() string {
- return fmt.Sprintf("%s:_sys:tables", m.database)
- }
- // rowIDKey returns the key for a table's next ROWID counter.
- func (m *SchemaManager) rowIDKey(table string) string {
- return fmt.Sprintf("%s:_sys:rowid:%s", m.database, strings.ToLower(table))
- }
- // CreateTable creates a new table.
- func (m *SchemaManager) CreateTable(schema *Schema) error {
- m.mu.Lock()
- defer m.mu.Unlock()
- // Check if table already exists
- key := m.schemaKey(schema.Name)
- err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.Read(key)
- return err
- })
- if err == nil {
- return fmt.Errorf("table already exists: %s", schema.Name)
- }
- // Keep the cached schema private so callers cannot mutate a published
- // catalog snapshot after this operation returns.
- schema = cloneSchema(schema)
- schema.CreatedAt = time.Now()
- // Determine primary key if not set
- if schema.PrimaryKey == "" {
- for _, col := range schema.Columns {
- if col.PrimaryKey {
- schema.PrimaryKey = col.Name
- break
- }
- }
- // No explicit primary key declared — use synthetic _rowid_ so user
- // columns remain unconstrained and can hold duplicate or NULL values.
- if schema.PrimaryKey == "" {
- schema.PrimaryKey = "_rowid_"
- }
- }
- // Serialize schema
- data, err := json.Marshal(schema)
- if err != nil {
- return fmt.Errorf("failed to serialize schema: %w", err)
- }
- // Write schema
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(data))
- })
- if err != nil {
- return fmt.Errorf("failed to write schema: %w", err)
- }
- // Update catalog
- if err := m.addToCatalog(schema.Name); err != nil {
- // Rollback schema write
- m.pool.WithClient(func(c *KVClient) error {
- return c.Delete(key)
- })
- return err
- }
- // Update cache
- m.cache[strings.ToLower(schema.Name)] = schema
- m.bumpVersionLocked()
- return nil
- }
- // DropTable drops a table.
- func (m *SchemaManager) DropTable(name string) error {
- tableLock := m.tableLock(name)
- tableLock.Lock()
- defer tableLock.Unlock()
- m.mu.Lock()
- defer m.mu.Unlock()
- key := m.schemaKey(name)
- // Check if table exists
- err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.Read(key)
- return err
- })
- if err != nil {
- return fmt.Errorf("table not found: %s", name)
- }
- // Delete all rows by scanning their actual keys and batch-deleting them, so
- // a table drop no longer leaks durable rows.
- if err := m.deleteKeysWithPrefix([]byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(name)))); err != nil {
- return err
- }
- // Delete schema
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Delete(key)
- })
- if err != nil {
- return fmt.Errorf("failed to delete schema: %w", err)
- }
- // Delete ROWID state.
- m.pool.WithClient(func(c *KVClient) error {
- return c.Delete(m.rowIDKey(name))
- })
- // Update catalog
- if err := m.removeFromCatalog(name); err != nil {
- return err
- }
- // Update cache
- tableLower := strings.ToLower(name)
- delete(m.cache, tableLower)
- m.rowIDMu.Lock()
- delete(m.rowIDAlloc, tableLower)
- m.rowIDMu.Unlock()
- m.bumpVersionLocked()
- return nil
- }
- // GetSchema retrieves a table schema.
- func (m *SchemaManager) GetSchema(name string) (*Schema, error) {
- m.mu.RLock()
- if schema, ok := m.cache[strings.ToLower(name)]; ok {
- // Clone while still holding the read lock: the cached schema's
- // NextRowID field is mutated under the write lock, so cloning outside
- // the lock races with that mutation.
- cloned := cloneSchema(schema)
- m.mu.RUnlock()
- return cloned, nil
- }
- m.mu.RUnlock()
- m.mu.Lock()
- defer m.mu.Unlock()
- // Double-check after acquiring write lock
- if schema, ok := m.cache[strings.ToLower(name)]; ok {
- return cloneSchema(schema), nil
- }
- key := m.schemaKey(name)
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err != nil {
- if err == ErrKeyNotFound {
- return nil, fmt.Errorf("table not found: %s", name)
- }
- return nil, err
- }
- var schema Schema
- if err := json.Unmarshal([]byte(data), &schema); err != nil {
- return nil, fmt.Errorf("failed to parse schema: %w", err)
- }
- m.cache[strings.ToLower(name)] = &schema
- return cloneSchema(&schema), nil
- }
- func cloneSchema(schema *Schema) *Schema {
- if schema == nil {
- return nil
- }
- cloned := *schema
- cloned.Columns = append([]Column(nil), schema.Columns...)
- cloned.Checks = append([]CheckConstraint(nil), schema.Checks...)
- return &cloned
- }
- func cloneIndex(index *Index) *Index {
- if index == nil {
- return nil
- }
- cloned := *index
- cloned.Columns = append([]IndexColumn(nil), index.Columns...)
- return &cloned
- }
- // TableExists checks if a table exists.
- func (m *SchemaManager) TableExists(name string) bool {
- _, err := m.GetSchema(name)
- return err == nil
- }
- // ListTables returns all table names.
- func (m *SchemaManager) ListTables() ([]string, error) {
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(m.catalogKey())
- return err
- })
- if err != nil {
- if err == ErrKeyNotFound {
- return nil, nil
- }
- return nil, err
- }
- var tables []string
- if err := json.Unmarshal([]byte(data), &tables); err != nil {
- return nil, fmt.Errorf("failed to parse catalog: %w", err)
- }
- return tables, nil
- }
- // addToCatalog adds a table to the catalog.
- func (m *SchemaManager) addToCatalog(name string) error {
- tables, err := m.ListTables()
- if err != nil && err != ErrKeyNotFound {
- return err
- }
- // Check if already exists
- lowerName := strings.ToLower(name)
- for _, t := range tables {
- if strings.ToLower(t) == lowerName {
- return nil
- }
- }
- tables = append(tables, name)
- data, err := json.Marshal(tables)
- if err != nil {
- return err
- }
- return m.pool.WithClient(func(c *KVClient) error {
- return c.Write(m.catalogKey(), string(data))
- })
- }
- // removeFromCatalog removes a table from the catalog.
- func (m *SchemaManager) removeFromCatalog(name string) error {
- tables, err := m.ListTables()
- if err != nil {
- return err
- }
- lowerName := strings.ToLower(name)
- newTables := make([]string, 0, len(tables))
- for _, t := range tables {
- if strings.ToLower(t) != lowerName {
- newTables = append(newTables, t)
- }
- }
- data, err := json.Marshal(newTables)
- if err != nil {
- return err
- }
- return m.pool.WithClient(func(c *KVClient) error {
- return c.Write(m.catalogKey(), string(data))
- })
- }
- // InvalidateCache clears the cache for a table.
- func (m *SchemaManager) InvalidateCache(name string) {
- m.mu.Lock()
- defer m.mu.Unlock()
- tableLower := strings.ToLower(name)
- delete(m.cache, tableLower)
- m.rowIDMu.Lock()
- delete(m.rowIDAlloc, tableLower)
- m.rowIDMu.Unlock()
- }
- // ToAnalyzerTableInfo converts a Schema to analyzer.TableInfo.
- func (s *Schema) ToAnalyzerTableInfo() *analyzer.TableInfo {
- info := &analyzer.TableInfo{
- Name: s.Name,
- }
- for _, col := range s.Columns {
- info.Columns = append(info.Columns, analyzer.ColumnInfo{
- Name: col.Name,
- Type: analyzer.TypeFromName(col.Type),
- Nullable: col.Nullable,
- PrimaryKey: col.PrimaryKey,
- TableName: s.Name,
- Generated: col.GeneratedExpr != "",
- })
- }
- return info
- }
- // GetColumn returns a column by name.
- func (s *Schema) GetColumn(name string) (*Column, bool) {
- lowerName := strings.ToLower(name)
- for i := range s.Columns {
- if strings.ToLower(s.Columns[i].Name) == lowerName {
- return &s.Columns[i], true
- }
- }
- return nil, false
- }
- // GetNextRowID gets and increments the next ROWID for a table. Allocation is
- // serialized per table via the table's own allocator so inserts on different
- // tables never contend, and no global SchemaManager lock is held across the
- // durable derivation scan.
- func (m *SchemaManager) GetNextRowID(table string) (int64, error) {
- alloc := m.rowIDAllocatorFor(table)
- alloc.mu.Lock()
- defer alloc.mu.Unlock()
- next, err := m.nextRowIDLocked(alloc, table)
- if err != nil {
- return 0, err
- }
- alloc.next = next + 1
- return next, nil
- }
- // UpdateMaxRowID updates the next ROWID if the provided value is higher.
- func (m *SchemaManager) UpdateMaxRowID(table string, rowid int64) error {
- alloc := m.rowIDAllocatorFor(table)
- alloc.mu.Lock()
- defer alloc.mu.Unlock()
- next, err := m.nextRowIDLocked(alloc, table)
- if err != nil {
- return err
- }
- if rowid >= next {
- alloc.next = rowid + 1
- }
- return nil
- }
- // nextRowIDLocked returns the current next ROWID, deriving it from durable rows
- // on first use (must hold the per-table allocator lock).
- func (m *SchemaManager) nextRowIDLocked(alloc *rowIDAllocator, table string) (int64, error) {
- if alloc.init {
- return alloc.next, nil
- }
- next, err := m.deriveNextRowID(table)
- if err != nil {
- return 0, err
- }
- if next < 1 {
- next = 1
- }
- alloc.next = next
- alloc.init = true
- return alloc.next, nil
- }
- // deriveNextRowID scans durable row keys to recover max(rowid)+1, streaming
- // each page through decodeRow instead of materializing every value.
- func (m *SchemaManager) deriveNextRowID(table string) (int64, error) {
- prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
- var maxRowID int64
- err := m.pool.WithClient(func(client *KVClient) (retErr error) {
- cursor, err := client.Scan(prefix)
- if err != nil {
- return err
- }
- defer func() {
- if err := cursor.Close(); retErr == nil {
- retErr = err
- }
- }()
- for {
- entries, done, err := cursor.Next()
- if err != nil {
- return err
- }
- for _, e := range entries {
- row, err := decodeRow(e.Value)
- if err != nil {
- return fmt.Errorf("failed to parse row while deriving ROWID: %w", err)
- }
- if rowid, ok := valueAsInt64(row["_rowid_"]); ok && rowid > maxRowID {
- maxRowID = rowid
- }
- }
- if done {
- return nil
- }
- }
- })
- if err != nil {
- return 0, err
- }
- return maxRowID + 1, nil
- }
- // deleteKeysWithPrefix streams the keys with the given prefix one page at a
- // time and atomically batch-deletes them, so a bulk operation never leaves
- // durable rows behind.
- func (m *SchemaManager) deleteKeysWithPrefix(prefix []byte) error {
- return m.pool.WithClient(func(client *KVClient) (retErr error) {
- cursor, err := client.ScanKeys(prefix)
- if err != nil {
- return err
- }
- defer func() {
- if err := cursor.Close(); retErr == nil {
- retErr = err
- }
- }()
- ops := make([]BatchOp, 0, scanPageSize)
- batchBytes := 8
- flush := func() error {
- if len(ops) == 0 {
- return nil
- }
- if _, err := client.BatchWrite(ops, nil); err != nil {
- return err
- }
- ops = ops[:0]
- batchBytes = 8
- return nil
- }
- for {
- entries, done, err := cursor.Next()
- if err != nil {
- return err
- }
- for _, e := range entries {
- opBytes := 12 + len(e.Key)
- if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
- if err := flush(); err != nil {
- return err
- }
- }
- ops = append(ops, BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)})
- batchBytes += opBytes
- }
- if done {
- return flush()
- }
- }
- })
- }
- func valueAsInt64(value interface{}) (int64, bool) {
- switch v := value.(type) {
- case int64:
- return v, true
- case int:
- return int64(v), true
- case float64:
- return int64(v), true
- default:
- return 0, false
- }
- }
- // getSchemaLocked retrieves schema (must hold lock).
- func (m *SchemaManager) getSchemaLocked(name string) (*Schema, error) {
- if schema, ok := m.cache[strings.ToLower(name)]; ok {
- return schema, nil
- }
- key := m.schemaKey(name)
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err != nil {
- if err == ErrKeyNotFound {
- return nil, fmt.Errorf("table not found: %s", name)
- }
- return nil, err
- }
- var schema Schema
- if err := json.Unmarshal([]byte(data), &schema); err != nil {
- return nil, fmt.Errorf("failed to parse schema: %w", err)
- }
- m.cache[strings.ToLower(name)] = &schema
- return &schema, nil
- }
- // saveSchemaLocked saves schema (must hold lock).
- func (m *SchemaManager) saveSchemaLocked(schema *Schema) error {
- data, err := json.Marshal(schema)
- if err != nil {
- return fmt.Errorf("failed to serialize schema: %w", err)
- }
- key := m.schemaKey(schema.Name)
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(data))
- })
- if err != nil {
- return fmt.Errorf("failed to write schema: %w", err)
- }
- m.cache[strings.ToLower(schema.Name)] = schema
- return nil
- }
- // Index management methods
- // indexKey returns the key for an index.
- func (m *SchemaManager) indexKey(name string) string {
- return fmt.Sprintf("%s:index:%s", m.database, strings.ToLower(name))
- }
- // indexListKey returns the key for the index list.
- func (m *SchemaManager) indexListKey() string {
- return fmt.Sprintf("%s:indexes", m.database)
- }
- // CreateIndex creates a new index.
- func (m *SchemaManager) CreateIndex(index *Index) error {
- m.mu.Lock()
- defer m.mu.Unlock()
- // Check if index already exists
- key := m.indexKey(index.Name)
- err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.Read(key)
- return err
- })
- if err == nil {
- return fmt.Errorf("index already exists: %s", index.Name)
- }
- // Verify table exists
- if _, err := m.getSchemaLocked(index.Table); err != nil {
- return fmt.Errorf("table not found: %s", index.Table)
- }
- // Save index
- index.CreatedAt = time.Now()
- data, err := json.Marshal(index)
- if err != nil {
- return fmt.Errorf("failed to serialize index: %w", err)
- }
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(data))
- })
- if err != nil {
- return fmt.Errorf("failed to write index: %w", err)
- }
- // Add to index list
- if err := m.addToIndexList(index.Name); err != nil {
- return err
- }
- m.indexCache[strings.ToLower(index.Name)] = cloneIndex(index)
- m.bumpVersionLocked()
- return nil
- }
- // DropIndex drops an index.
- func (m *SchemaManager) DropIndex(name string) error {
- m.mu.Lock()
- defer m.mu.Unlock()
- key := m.indexKey(name)
- err := m.pool.WithClient(func(c *KVClient) error {
- return c.Delete(key)
- })
- if err != nil {
- return fmt.Errorf("failed to delete index: %w", err)
- }
- if err := m.removeFromIndexList(name); err != nil {
- return err
- }
- delete(m.indexCache, strings.ToLower(name))
- m.bumpVersionLocked()
- return nil
- }
- // IndexExists checks if an index exists.
- func (m *SchemaManager) IndexExists(name string) bool {
- m.mu.RLock()
- defer m.mu.RUnlock()
- return m.indexExistsUnsafe(name)
- }
- // indexExistsUnsafe reports whether an index definition is durable (m.mu must be
- // held).
- func (m *SchemaManager) indexExistsUnsafe(name string) bool {
- if _, ok := m.indexCache[strings.ToLower(name)]; ok {
- return true
- }
- key := m.indexKey(name)
- err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.Read(key)
- return err
- })
- return err == nil
- }
- // GetIndex retrieves an index by name.
- func (m *SchemaManager) GetIndex(name string) (*Index, error) {
- m.mu.Lock()
- defer m.mu.Unlock()
- cacheKey := strings.ToLower(name)
- if index, ok := m.indexCache[cacheKey]; ok {
- return cloneIndex(index), nil
- }
- key := m.indexKey(name)
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err != nil {
- if err == ErrKeyNotFound {
- return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
- }
- return nil, err
- }
- var index Index
- if err := json.Unmarshal([]byte(data), &index); err != nil {
- return nil, fmt.Errorf("failed to parse index: %w", err)
- }
- m.indexCache[cacheKey] = &index
- return cloneIndex(&index), nil
- }
- // ListIndexes returns all index names.
- func (m *SchemaManager) ListIndexes() ([]string, error) {
- m.mu.Lock()
- defer m.mu.Unlock()
- return m.indexListUnsafe()
- }
- // indexListUnsafe returns all index names, loading them from storage on first
- // use (m.mu must be held).
- func (m *SchemaManager) indexListUnsafe() ([]string, error) {
- if m.indexListCached {
- return append([]string(nil), m.indexListCache...), nil
- }
- key := m.indexListKey()
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err == ErrKeyNotFound {
- m.indexListCache = nil
- m.indexListCached = true
- return []string{}, nil
- }
- if err != nil {
- return nil, err
- }
- var indexes []string
- if err := json.Unmarshal([]byte(data), &indexes); err != nil {
- return nil, fmt.Errorf("failed to parse index list: %w", err)
- }
- m.indexListCache = append([]string(nil), indexes...)
- m.indexListCached = true
- return indexes, nil
- }
- // ListTableIndexes returns all indexes for a table.
- func (m *SchemaManager) ListTableIndexes(table string) ([]*Index, error) {
- indexes, err := m.ListIndexes()
- if err != nil {
- return nil, err
- }
- var result []*Index
- for _, name := range indexes {
- idx, err := m.GetIndex(name)
- if err != nil {
- // An index dropped concurrently is simply absent; any other error
- // (storage/IO or corrupt data) must not be silently swallowed.
- if errors.Is(err, ErrIndexNotFound) {
- continue
- }
- return nil, err
- }
- if strings.EqualFold(idx.Table, table) {
- result = append(result, idx)
- }
- }
- return result, nil
- }
- // addToIndexList adds an index name to the list.
- func (m *SchemaManager) addToIndexList(name string) error {
- key := m.indexListKey()
- var indexes []string
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err == nil {
- json.Unmarshal([]byte(data), &indexes)
- }
- indexes = append(indexes, name)
- newData, _ := json.Marshal(indexes)
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(newData))
- })
- if err == nil {
- m.indexListCache = append([]string(nil), indexes...)
- m.indexListCached = true
- }
- return err
- }
- // removeFromIndexList removes an index name from the list.
- func (m *SchemaManager) removeFromIndexList(name string) error {
- key := m.indexListKey()
- var indexes []string
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err != nil {
- return nil
- }
- json.Unmarshal([]byte(data), &indexes)
- var newIndexes []string
- for _, idx := range indexes {
- if !strings.EqualFold(idx, name) {
- newIndexes = append(newIndexes, idx)
- }
- }
- newData, _ := json.Marshal(newIndexes)
- err = m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(newData))
- })
- if err == nil {
- m.indexListCache = append([]string(nil), newIndexes...)
- m.indexListCached = true
- }
- return err
- }
- // AddColumn adds a new column to a table. When unique is non-nil the column's
- // UNIQUE index is registered as part of the same statement, and checks are
- // persisted with the column in the same schema snapshot.
- //
- // A constant DEFAULT is materialized into every existing row before the schema
- // is published. NOT NULL, UNIQUE, CHECK, and index validation all read
- // persisted row values, so a default that only existed in the catalog would be
- // invisible to them: a UNIQUE index over the new column could not see the
- // duplicate values every existing row shares, a CHECK predicate could not see a
- // violating default, and an UPDATE to NULL would look non-null. The rewrite is
- // the same shape as DropColumn: the table's exclusive gate keeps point writes
- // and CAS commits out while rows are scanned, validated, and rewritten, the
- // transform is idempotent, and the schema is saved last so an interrupted
- // operation resumes against the old schema.
- //
- // The gate also makes the validation authoritative: NOT NULL without a default,
- // duplicate UNIQUE values, and rows a new CHECK predicate rejects are rejected
- // before any row is touched, and the column, its checks, and its index are
- // published together in one batch write, so a crash cannot leave the column
- // visible without the constraints that validate it. A nullable column without a
- // default needs no rewrite because a missing row field already reads as NULL.
- // Like DROP COLUMN, the statement holds the catalog lock for the whole scan, so
- // the cost is one table scan per validating constraint plus a table-wide stall
- // per ADD COLUMN that has a default, an index, or a check.
- func (m *SchemaManager) AddColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
- tableLock := m.tableLock(table)
- tableLock.Lock()
- defer tableLock.Unlock()
- m.mu.Lock()
- defer m.mu.Unlock()
- schema, err := m.getSchemaUnsafe(table)
- if err != nil {
- return err
- }
- if err := checkColumnAbsent(schema, column.Name); err != nil {
- return err
- }
- if unique != nil {
- if !strings.EqualFold(unique.Table, schema.Name) {
- return fmt.Errorf("index %s belongs to table %s, not %s", unique.Name, unique.Table, schema.Name)
- }
- if m.indexExistsUnsafe(unique.Name) {
- return fmt.Errorf("index already exists: %s", unique.Name)
- }
- if err := m.validateAddColumnUnique(table, column, unique); err != nil {
- return err
- }
- }
- if err := m.validateAddColumnChecks(table, column, checks); err != nil {
- return err
- }
- // A required column with nothing to write into rows that already exist
- // cannot be declared on a populated table.
- if !column.Nullable && column.Default == nil {
- hasRows, err := m.tableHasRows(table)
- if err != nil {
- return err
- }
- if hasRows {
- return &NotNullViolationError{Table: schema.Name, Column: column.Name}
- }
- }
- if column.Default != nil {
- if err := m.rewriteRows(table, func(row Row) bool {
- if _, ok := rowColumnValue(row, column.Name); ok {
- return false
- }
- row[column.Name] = column.Default
- return true
- }); err != nil {
- return err
- }
- }
- return m.publishAddedColumn(schema, column, checks, unique)
- }
- // publishAddedColumn publishes a column, its checks, and, when present, its
- // unique index in one atomic batch write (m.mu must be held): the schema
- // snapshot, the index definition, and the index list always become durable
- // together or not at all.
- func (m *SchemaManager) publishAddedColumn(schema *Schema, column Column, checks []CheckConstraint, unique *Index) error {
- // Publish a fresh snapshot instead of mutating readers' shared pointer.
- schema = cloneSchema(schema)
- schema.Columns = append(schema.Columns, column)
- schema.Checks = append(schema.Checks, checks...)
- schemaData, err := json.Marshal(schema)
- if err != nil {
- return fmt.Errorf("failed to serialize schema: %w", err)
- }
- ops := []BatchOp{{Op: batchPut, Key: []byte(m.schemaKey(schema.Name)), Value: schemaData}}
- var indexList []string
- if unique != nil {
- unique.CreatedAt = time.Now()
- indexData, err := json.Marshal(unique)
- if err != nil {
- return fmt.Errorf("failed to serialize index: %w", err)
- }
- indexList, err = m.indexListUnsafe()
- if err != nil {
- return err
- }
- indexList = append(indexList, unique.Name)
- listData, err := json.Marshal(indexList)
- if err != nil {
- return fmt.Errorf("failed to serialize index list: %w", err)
- }
- ops = append(ops,
- BatchOp{Op: batchPut, Key: []byte(m.indexKey(unique.Name)), Value: indexData},
- BatchOp{Op: batchPut, Key: []byte(m.indexListKey()), Value: listData},
- )
- }
- if err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.BatchWrite(ops, nil)
- return err
- }); err != nil {
- return fmt.Errorf("failed to publish column %s: %w", column.Name, err)
- }
- // Publish in-memory state only after the durable write.
- m.cache[strings.ToLower(schema.Name)] = schema
- if unique != nil {
- m.indexCache[strings.ToLower(unique.Name)] = cloneIndex(unique)
- m.indexListCache = append([]string(nil), indexList...)
- m.indexListCached = true
- }
- m.bumpVersionLocked()
- return nil
- }
- func checkColumnAbsent(schema *Schema, name string) error {
- for _, col := range schema.Columns {
- if strings.EqualFold(col.Name, name) {
- return fmt.Errorf("column already exists: %s", name)
- }
- }
- return nil
- }
- // DropColumn removes a column from a table.
- func (m *SchemaManager) DropColumn(table, columnName string) error {
- tableLock := m.tableLock(table)
- tableLock.Lock()
- defer tableLock.Unlock()
- m.mu.Lock()
- defer m.mu.Unlock()
- schema, err := m.getSchemaUnsafe(table)
- if err != nil {
- return err
- }
- // Cannot drop primary key column
- if strings.EqualFold(schema.PrimaryKey, columnName) {
- return fmt.Errorf("cannot drop primary key column: %s", columnName)
- }
- // Find and remove column
- newColumns := make([]Column, 0, len(schema.Columns)-1)
- found := false
- for _, col := range schema.Columns {
- if strings.EqualFold(col.Name, columnName) {
- found = true
- continue
- }
- newColumns = append(newColumns, col)
- }
- if !found {
- return fmt.Errorf("column not found: %s", columnName)
- }
- schema = cloneSchema(schema)
- schema.Columns = newColumns
- if err := m.rewriteRows(table, func(row Row) bool {
- for key := range row {
- if strings.EqualFold(key, columnName) {
- delete(row, key)
- return true
- }
- }
- return false
- }); err != nil {
- return err
- }
- // Update schema
- return m.updateSchemaUnsafe(schema)
- }
- // RenameTable renames a table.
- func (m *SchemaManager) RenameTable(oldName, newName string) error {
- m.mu.Lock()
- defer m.mu.Unlock()
- // Check if old table exists
- schema, err := m.getSchemaUnsafe(oldName)
- if err != nil {
- return err
- }
- // Check if new table name already exists
- _, err = m.getSchemaUnsafe(newName)
- if err == nil {
- return fmt.Errorf("table already exists: %s", newName)
- }
- schema = cloneSchema(schema)
- // Update schema name
- schema.Name = newName
- // Move durable rows from the old table's data prefix to the new one. Without
- // this, ALTER TABLE ... RENAME leaves every row under the old key and the
- // renamed table appears empty.
- if err := m.renameDataKeys(oldName, newName); err != nil {
- return err
- }
- // Repoint indexes that belonged to the old table.
- if err := m.repointIndexesLocked(oldName, newName); err != nil {
- return err
- }
- // Build the replacement catalog before switching schema names. The schema,
- // catalog, and stale ROWID state change in one batch, so a crash cannot
- // leave neither table name addressable after all row chunks have moved.
- tables, err := m.ListTables()
- if err != nil {
- return err
- }
- foundOld := false
- for i, name := range tables {
- if strings.EqualFold(name, oldName) {
- tables[i] = newName
- foundOld = true
- break
- }
- }
- if !foundOld {
- return fmt.Errorf("table not found in catalog: %s", oldName)
- }
- catalogData, err := json.Marshal(tables)
- if err != nil {
- return err
- }
- schemaData, err := json.Marshal(schema)
- if err != nil {
- return err
- }
- oldKey := m.schemaKey(oldName)
- newKey := m.schemaKey(newName)
- err = m.pool.WithClient(func(c *KVClient) error {
- _, err := c.BatchWrite([]BatchOp{
- {Op: batchPut, Key: []byte(newKey), Value: schemaData},
- {Op: batchDelete, Key: []byte(oldKey)},
- {Op: batchDelete, Key: []byte(m.rowIDKey(oldName))},
- {Op: batchPut, Key: []byte(m.catalogKey()), Value: catalogData},
- }, nil)
- return err
- })
- if err != nil {
- return err
- }
- // Update cache
- oldLower := strings.ToLower(oldName)
- newLower := strings.ToLower(newName)
- delete(m.cache, oldLower)
- m.rowIDMu.Lock()
- delete(m.rowIDAlloc, oldLower)
- m.rowIDMu.Unlock()
- // Update cache
- m.cache[newLower] = schema
- m.bumpVersionLocked()
- return nil
- }
- // renameDataKeys moves every durable row of a table from the old name's data
- // prefix to the new name's prefix in a single atomic batch per page.
- func (m *SchemaManager) renameDataKeys(oldName, newName string) error {
- oldPrefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(oldName)))
- newPrefix := fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(newName))
- return m.pool.WithClient(func(client *KVClient) (retErr error) {
- cursor, err := client.Scan(oldPrefix)
- if err != nil {
- return err
- }
- defer func() {
- if cerr := cursor.Close(); retErr == nil {
- retErr = cerr
- }
- }()
- ops := make([]BatchOp, 0, scanPageSize*2)
- batchBytes := 8
- flush := func() error {
- if len(ops) == 0 {
- return nil
- }
- if _, err := client.BatchWrite(ops, nil); err != nil {
- return err
- }
- ops = ops[:0]
- batchBytes = 8
- return nil
- }
- for {
- entries, done, err := cursor.Next()
- if err != nil {
- return err
- }
- for _, e := range entries {
- suffix := string(e.Key[len(oldPrefix):])
- newKey := []byte(newPrefix + suffix)
- opBytes := 24 + len(newKey) + len(e.Value) + len(e.Key)
- if len(ops)+2 > maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
- if err := flush(); err != nil {
- return err
- }
- }
- ops = append(ops,
- BatchOp{Op: batchPut, Key: newKey, Value: e.Value},
- BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)},
- )
- batchBytes += opBytes
- }
- if done {
- return flush()
- }
- }
- })
- }
- // updateIndexesOfTableLocked applies mutate to the durable definition of every
- // index on table and persists the definitions it changed. mutate reports
- // whether it modified the definition; unmodified indexes are left untouched.
- // The changed definitions are written in one batch, so a table's indexes never
- // split between the old and new shape. The caller holds m.mu; it reads the
- // durable index list directly rather than calling the self-locking ListIndexes
- // helper.
- func (m *SchemaManager) updateIndexesOfTableLocked(table string, mutate func(*Index) bool) error {
- var names []string
- if m.indexListCached {
- names = append([]string(nil), m.indexListCache...)
- } else {
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var rerr error
- data, rerr = c.Read(m.indexListKey())
- return rerr
- })
- if err == ErrKeyNotFound {
- return nil
- }
- if err != nil {
- return err
- }
- if err := json.Unmarshal([]byte(data), &names); err != nil {
- return err
- }
- }
- type indexUpdate struct {
- index Index
- key string
- value []byte
- }
- var updates []indexUpdate
- for _, name := range names {
- key := m.indexKey(name)
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var rerr error
- data, rerr = c.Read(key)
- return rerr
- })
- if err != nil {
- if err == ErrKeyNotFound {
- continue
- }
- return err
- }
- var idx Index
- if err := json.Unmarshal([]byte(data), &idx); err != nil {
- return err
- }
- if !strings.EqualFold(idx.Table, table) {
- continue
- }
- if !mutate(&idx) {
- continue
- }
- updated, err := json.Marshal(&idx)
- if err != nil {
- return err
- }
- updates = append(updates, indexUpdate{index: idx, key: key, value: updated})
- }
- if len(updates) == 0 {
- return nil
- }
- ops := make([]BatchOp, 0, len(updates))
- for _, update := range updates {
- ops = append(ops, BatchOp{Op: batchPut, Key: []byte(update.key), Value: update.value})
- }
- if err := m.pool.WithClient(func(c *KVClient) error {
- _, err := c.BatchWrite(ops, nil)
- return err
- }); err != nil {
- return fmt.Errorf("update index definitions for table %s: %w", table, err)
- }
- for _, update := range updates {
- if cached, ok := m.indexCache[strings.ToLower(update.index.Name)]; ok {
- *cached = update.index
- }
- }
- return nil
- }
- // repointIndexesLocked updates indexes whose table was renamed.
- func (m *SchemaManager) repointIndexesLocked(oldName, newName string) error {
- return m.updateIndexesOfTableLocked(oldName, func(idx *Index) bool {
- idx.Table = newName
- return true
- })
- }
- // renameIndexColumnsLocked rewrites the stored definition of every index on
- // table so a reference to oldName becomes newName. Index names stay unchanged:
- // they are independent identifiers that SQLite does not derive from column
- // names, and callers may have chosen them explicitly. A rename to an existing
- // column name is rejected before this runs. Expression columns keep their SQL
- // text; callers must reject a rename that such an expression depends on,
- // because the text is not rewritten.
- func (m *SchemaManager) renameIndexColumnsLocked(table, oldName, newName string) error {
- return m.updateIndexesOfTableLocked(table, func(idx *Index) bool {
- changed := false
- for i := range idx.Columns {
- // Expression columns persist SQL text, not a column reference; their
- // text is left for the caller to drop and recreate the index.
- if idx.Columns[i].Expression == "" && strings.EqualFold(idx.Columns[i].Name, oldName) {
- idx.Columns[i].Name = newName
- changed = true
- }
- }
- return changed
- })
- }
- // RenameColumn renames a column in a table. Column references in index
- // definitions are rewritten; expression-index text is not, so a SQL caller must
- // reject a rename that an expression depends on.
- func (m *SchemaManager) RenameColumn(table, oldName, newName string) error {
- tableLock := m.tableLock(table)
- tableLock.Lock()
- defer tableLock.Unlock()
- m.mu.Lock()
- defer m.mu.Unlock()
- schema, err := m.getSchemaUnsafe(table)
- if err != nil {
- return err
- }
- // Check if new column name already exists
- for _, col := range schema.Columns {
- if strings.EqualFold(col.Name, newName) {
- return fmt.Errorf("column already exists: %s", newName)
- }
- }
- schema = cloneSchema(schema)
- // Find and rename column
- found := false
- for i, col := range schema.Columns {
- if strings.EqualFold(col.Name, oldName) {
- schema.Columns[i].Name = newName
- found = true
- // Update primary key reference if needed
- if strings.EqualFold(schema.PrimaryKey, oldName) {
- schema.PrimaryKey = newName
- }
- break
- }
- }
- if !found {
- return fmt.Errorf("column not found: %s", oldName)
- }
- if err := m.rewriteRows(table, func(row Row) bool {
- for key, value := range row {
- if strings.EqualFold(key, oldName) {
- delete(row, key)
- row[newName] = value
- return true
- }
- }
- return false
- }); err != nil {
- return err
- }
- // Publish the renamed schema before the index definitions: reads of the new
- // column name resolve against rewritten rows as soon as the schema lands.
- // Index DDL is not transactional (as elsewhere in this engine), so a failure
- // here leaves stale index metadata for the caller to drop and recreate.
- if err := m.updateSchemaUnsafe(schema); err != nil {
- return err
- }
- // Index metadata drives secondary-index values, UNIQUE enforcement,
- // ON CONFLICT target resolution, and catalog introspection, so it must name
- // the renamed column before any later statement uses the table.
- return m.renameIndexColumnsLocked(table, oldName, newName)
- }
- // rewriteRows applies an idempotent name-keyed row transformation in bounded
- // batches. Column DDL updates the schema only after all rows are rewritten, so
- // an interrupted operation can safely resume against the old schema.
- func (m *SchemaManager) rewriteRows(table string, transform func(Row) bool) error {
- return m.applyRows(table, func(row Row) (bool, error) {
- return transform(row), nil
- })
- }
- // applyRows scans a table once and applies transform to every row. Rows for
- // which transform reports a change are re-encoded and written in bounded
- // batches; an error from transform aborts the scan. Callers hold the table's
- // exclusive gate, so no writer can interleave between the scan and the schema
- // update that follows it. Aborting after a batch was flushed is safe only
- // because the DDL transforms are idempotent.
- func (m *SchemaManager) applyRows(table string, transform func(Row) (bool, error)) error {
- prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
- return m.pool.WithClient(func(client *KVClient) (retErr error) {
- cursor, err := client.Scan(prefix)
- if err != nil {
- return err
- }
- defer func() {
- if cerr := cursor.Close(); retErr == nil {
- retErr = cerr
- }
- }()
- ops := make([]BatchOp, 0, scanPageSize)
- batchBytes := 8
- flush := func() error {
- if len(ops) == 0 {
- return nil
- }
- if _, err := client.BatchWrite(ops, nil); err != nil {
- return err
- }
- ops = ops[:0]
- batchBytes = 8
- return nil
- }
- for {
- entries, done, err := cursor.Next()
- if err != nil {
- return err
- }
- for _, entry := range entries {
- row, err := decodeRow(entry.Value)
- if err != nil {
- return err
- }
- changed, err := transform(row)
- if err != nil {
- return err
- }
- if !changed {
- continue
- }
- value, err := encodeRow(row)
- if err != nil {
- return err
- }
- opBytes := 12 + len(entry.Key) + len(value)
- if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
- if err := flush(); err != nil {
- return err
- }
- }
- ops = append(ops, BatchOp{Op: batchPut, Key: append([]byte(nil), entry.Key...), Value: value})
- batchBytes += opBytes
- }
- if done {
- return flush()
- }
- }
- })
- }
- // validateAddColumnUnique rejects an ADD COLUMN ... UNIQUE whose backfilled
- // values would duplicate across existing rows. It reads rows as they will look
- // after the rewrite (a row that already carries the field from an interrupted
- // earlier attempt keeps its value) and mirrors ValidateUniqueIndex for a single
- // plain column: NULLs do not collide. The caller holds the table's exclusive
- // gate, so no writer can insert a conflicting row between this check and the
- // publication of the index.
- func (m *SchemaManager) validateAddColumnUnique(table string, column Column, index *Index) error {
- seen := make(map[string]struct{})
- return m.applyRows(table, func(row Row) (bool, error) {
- value, ok := rowColumnValue(row, column.Name)
- if !ok {
- value = column.Default
- }
- if value == nil {
- return false, nil
- }
- encoded := encodeUniqueScalar(value)
- if _, duplicate := seen[encoded]; duplicate {
- return false, &UniqueViolationError{Constraint: index.Name}
- }
- seen[encoded] = struct{}{}
- return false, nil
- })
- }
- // validateAddColumnChecks rejects an ADD COLUMN ... CHECK whose predicate is
- // already false for an existing row. Like validateAddColumnUnique it reads rows
- // as they will look after the constant-default backfill, and it runs before that
- // rewrite so a rejected DDL leaves no row carrying the new field. The caller
- // holds the table's exclusive gate, so no writer can insert a violating row
- // between this validation and the publication of the check.
- func (m *SchemaManager) validateAddColumnChecks(table string, column Column, checks []CheckConstraint) error {
- if len(checks) == 0 {
- return nil
- }
- eval := m.checkEvaluator()
- if eval == nil {
- return fmt.Errorf("cannot validate CHECK constraints on %s: no CHECK evaluator installed", table)
- }
- return m.applyRows(table, func(row Row) (bool, error) {
- if _, ok := rowColumnValue(row, column.Name); !ok && column.Default != nil {
- row[column.Name] = column.Default
- }
- for _, check := range checks {
- holds, err := eval(check.Expression, row)
- if err != nil {
- return false, fmt.Errorf("evaluate CHECK constraint %s on %s: %w", check.Label(), table, err)
- }
- if !holds {
- return false, &CheckViolationError{Table: table, Constraint: check.Label()}
- }
- }
- return false, nil
- })
- }
- // tableHasRows reports whether a table holds at least one durable row.
- func (m *SchemaManager) tableHasRows(table string) (bool, error) {
- prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
- found := false
- err := m.pool.WithClient(func(client *KVClient) (retErr error) {
- cursor, err := client.ScanWithLimit(prefix, 1)
- if err != nil {
- return err
- }
- defer func() {
- if cerr := cursor.Close(); retErr == nil {
- retErr = cerr
- }
- }()
- entries, _, err := cursor.Next()
- if err != nil {
- return err
- }
- found = len(entries) > 0
- return nil
- })
- return found, err
- }
- // getSchemaUnsafe gets a schema without locking (internal use).
- func (m *SchemaManager) getSchemaUnsafe(table string) (*Schema, error) {
- tableLower := strings.ToLower(table)
- // Check cache
- if schema, ok := m.cache[tableLower]; ok {
- return schema, nil
- }
- // Read from storage
- key := m.schemaKey(table)
- var data string
- err := m.pool.WithClient(func(c *KVClient) error {
- var err error
- data, err = c.Read(key)
- return err
- })
- if err != nil {
- return nil, fmt.Errorf("table not found: %s", table)
- }
- var schema Schema
- if err := json.Unmarshal([]byte(data), &schema); err != nil {
- return nil, err
- }
- m.cache[tableLower] = &schema
- return &schema, nil
- }
- // updateSchemaUnsafe updates a schema without locking (internal use).
- func (m *SchemaManager) updateSchemaUnsafe(schema *Schema) error {
- key := m.schemaKey(schema.Name)
- data, _ := json.Marshal(schema)
- err := m.pool.WithClient(func(c *KVClient) error {
- return c.Write(key, string(data))
- })
- if err != nil {
- return err
- }
- // Update cache
- m.cache[strings.ToLower(schema.Name)] = schema
- m.bumpVersionLocked()
- return nil
- }
|