schema.go 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728
  1. package storage
  2. import (
  3. "errors"
  4. "fmt"
  5. "strings"
  6. "sync"
  7. "time"
  8. "github.com/goccy/go-json"
  9. "github.com/danfragoso/pizzasql-next/pkg/analyzer"
  10. )
  11. // ErrIndexNotFound is returned by GetIndex when an index is absent. It is
  12. // distinct from a storage/IO error so callers like ListTableIndexes can skip a
  13. // concurrently-dropped index without swallowing real read failures.
  14. var ErrIndexNotFound = errors.New("index not found")
  15. // Schema represents a table schema.
  16. type Schema struct {
  17. Name string `json:"name"`
  18. Columns []Column `json:"columns"`
  19. PrimaryKey string `json:"primary_key"`
  20. CreatedAt time.Time `json:"created_at"`
  21. NextRowID int64 `json:"next_rowid"`
  22. AutoIncrement bool `json:"autoincrement"`
  23. // Checks holds the row predicates declared with CHECK, at column and table
  24. // level alike. Inline and table-level declarations are equivalent once
  25. // parsed, so both live in this one list.
  26. Checks []CheckConstraint `json:"checks,omitempty"`
  27. }
  28. // CheckConstraint is a persisted CHECK predicate. The predicate is stored as
  29. // SQL text and re-parsed by the evaluator installed on the SchemaManager, so a
  30. // durable reload keeps enforcing it.
  31. type CheckConstraint struct {
  32. Name string `json:"name,omitempty"`
  33. Expression string `json:"expression"`
  34. }
  35. // Label identifies the constraint in error messages and the catalog: the
  36. // declared name when there is one, otherwise the expression text.
  37. func (c CheckConstraint) Label() string {
  38. if c.Name != "" {
  39. return c.Name
  40. }
  41. return c.Expression
  42. }
  43. // Column represents a column definition.
  44. type Column struct {
  45. Name string `json:"name"`
  46. Type string `json:"type"`
  47. Nullable bool `json:"nullable"`
  48. Default interface{} `json:"default,omitempty"`
  49. PrimaryKey bool `json:"primary_key"`
  50. // GeneratedExpr holds the SQL text of a GENERATED ALWAYS AS (expr) column.
  51. // GeneratedStored reports whether the value is materialized on write
  52. // (STORED, the only form this engine persists) versus computed on read.
  53. GeneratedExpr string `json:"generated_expr,omitempty"`
  54. GeneratedStored bool `json:"generated_stored,omitempty"`
  55. }
  56. // Index represents an index definition.
  57. type Index struct {
  58. Name string `json:"name"`
  59. Table string `json:"table"`
  60. Columns []IndexColumn `json:"columns"`
  61. Unique bool `json:"unique"`
  62. CreatedAt time.Time `json:"created_at"`
  63. // OnConflict is the default conflict resolution declared for this index via
  64. // a UNIQUE(...) ON CONFLICT clause. Empty means the SQLite default (ABORT).
  65. OnConflict string `json:"on_conflict,omitempty"`
  66. }
  67. // IndexColumn represents a column in an index. Expression is set for expression
  68. // indexes (e.g. lower(email)); a plain column index leaves it empty and uses
  69. // Name.
  70. type IndexColumn struct {
  71. Name string `json:"name"`
  72. Desc bool `json:"desc"`
  73. Expression string `json:"expression,omitempty"`
  74. }
  75. // rowIDAllocator owns the next-ROWID state for a single table. It is a separate
  76. // mutex per table so allocating a ROWID on one table never serializes against
  77. // another table, and never contends with the SchemaManager catalog lock.
  78. type rowIDAllocator struct {
  79. mu sync.Mutex
  80. next int64
  81. init bool
  82. }
  83. // SchemaManager manages table schemas.
  84. type SchemaManager struct {
  85. pool *KVPool
  86. database string
  87. cache map[string]*Schema
  88. indexCache map[string]*Index
  89. indexListCache []string
  90. indexListCached bool
  91. version uint64
  92. mu sync.RWMutex
  93. tableLocksMu sync.Mutex
  94. tableLocks map[string]*sync.RWMutex
  95. // checkMu guards checkEval, the pluggable evaluator for persisted CHECK
  96. // predicates. It is set by the SQL executor so storage stays independent of
  97. // the SQL front end.
  98. checkMu sync.RWMutex
  99. checkEval CheckEvaluator
  100. rowIDMu sync.Mutex
  101. rowIDAlloc map[string]*rowIDAllocator
  102. }
  103. // CheckEvaluator reports whether a persisted CHECK expression holds for a row.
  104. // A false result is a violation; NULL and other unknown results must be
  105. // reported as true, matching SQL three-valued logic. The SQL executor registers
  106. // one so the storage layer can enforce CHECK without importing the parser.
  107. type CheckEvaluator func(expression string, row Row) (bool, error)
  108. // SetCheckEvaluator installs the evaluator used to enforce persisted CHECK
  109. // predicates.
  110. func (m *SchemaManager) SetCheckEvaluator(eval CheckEvaluator) {
  111. m.checkMu.Lock()
  112. m.checkEval = eval
  113. m.checkMu.Unlock()
  114. }
  115. func (m *SchemaManager) checkEvaluator() CheckEvaluator {
  116. m.checkMu.RLock()
  117. defer m.checkMu.RUnlock()
  118. return m.checkEval
  119. }
  120. // BeginTransaction is retained for API compatibility. Buffered per-session
  121. // transactions no longer take a database-wide transaction lock; staged writes
  122. // are validated and committed atomically with CompareBatchWrite instead.
  123. func (m *SchemaManager) BeginTransaction() {}
  124. // EndTransaction is retained for API compatibility.
  125. func (m *SchemaManager) EndTransaction() {}
  126. // LockStatement is retained for API compatibility. Statement execution is now
  127. // serialized through per-table locks and optimistic validation, so no global
  128. // statement lock is required.
  129. func (m *SchemaManager) LockStatement() {}
  130. // UnlockStatement is retained for API compatibility.
  131. func (m *SchemaManager) UnlockStatement() {}
  132. // NewSchemaManager creates a new schema manager.
  133. func NewSchemaManager(pool *KVPool, database string) *SchemaManager {
  134. return &SchemaManager{
  135. pool: pool,
  136. database: database,
  137. cache: make(map[string]*Schema),
  138. indexCache: make(map[string]*Index),
  139. tableLocks: make(map[string]*sync.RWMutex),
  140. rowIDAlloc: make(map[string]*rowIDAllocator),
  141. }
  142. }
  143. // rowIDAllocatorFor returns (creating if needed) the per-table ROWID allocator.
  144. func (m *SchemaManager) rowIDAllocatorFor(table string) *rowIDAllocator {
  145. key := strings.ToLower(table)
  146. m.rowIDMu.Lock()
  147. a, ok := m.rowIDAlloc[key]
  148. if !ok {
  149. a = &rowIDAllocator{}
  150. m.rowIDAlloc[key] = a
  151. }
  152. m.rowIDMu.Unlock()
  153. return a
  154. }
  155. func (m *SchemaManager) tableLock(table string) *sync.RWMutex {
  156. key := strings.ToLower(table)
  157. m.tableLocksMu.Lock()
  158. lock, ok := m.tableLocks[key]
  159. if !ok {
  160. lock = &sync.RWMutex{}
  161. m.tableLocks[key] = lock
  162. }
  163. m.tableLocksMu.Unlock()
  164. return lock
  165. }
  166. // GetDatabaseName returns the database name.
  167. func (m *SchemaManager) GetDatabaseName() string {
  168. return m.database
  169. }
  170. // GetPool returns the KV pool.
  171. func (m *SchemaManager) GetPool() *KVPool {
  172. return m.pool
  173. }
  174. // Version returns the in-process schema catalog version. It is incremented for
  175. // schema/index definition changes so cached executors can resync their analyzer
  176. // catalogs without scanning storage on every query.
  177. func (m *SchemaManager) Version() uint64 {
  178. m.mu.RLock()
  179. defer m.mu.RUnlock()
  180. return m.version
  181. }
  182. func (m *SchemaManager) bumpVersionLocked() {
  183. m.version++
  184. }
  185. // schemaKey returns the key for a table schema.
  186. func (m *SchemaManager) schemaKey(table string) string {
  187. return fmt.Sprintf("%s:_schema:%s", m.database, strings.ToLower(table))
  188. }
  189. // catalogKey returns the key for the table catalog.
  190. func (m *SchemaManager) catalogKey() string {
  191. return fmt.Sprintf("%s:_sys:tables", m.database)
  192. }
  193. // rowIDKey returns the key for a table's next ROWID counter.
  194. func (m *SchemaManager) rowIDKey(table string) string {
  195. return fmt.Sprintf("%s:_sys:rowid:%s", m.database, strings.ToLower(table))
  196. }
  197. // CreateTable creates a new table.
  198. func (m *SchemaManager) CreateTable(schema *Schema) error {
  199. m.mu.Lock()
  200. defer m.mu.Unlock()
  201. // Check if table already exists
  202. key := m.schemaKey(schema.Name)
  203. err := m.pool.WithClient(func(c *KVClient) error {
  204. _, err := c.Read(key)
  205. return err
  206. })
  207. if err == nil {
  208. return fmt.Errorf("table already exists: %s", schema.Name)
  209. }
  210. // Keep the cached schema private so callers cannot mutate a published
  211. // catalog snapshot after this operation returns.
  212. schema = cloneSchema(schema)
  213. schema.CreatedAt = time.Now()
  214. // Determine primary key if not set
  215. if schema.PrimaryKey == "" {
  216. for _, col := range schema.Columns {
  217. if col.PrimaryKey {
  218. schema.PrimaryKey = col.Name
  219. break
  220. }
  221. }
  222. // No explicit primary key declared — use synthetic _rowid_ so user
  223. // columns remain unconstrained and can hold duplicate or NULL values.
  224. if schema.PrimaryKey == "" {
  225. schema.PrimaryKey = "_rowid_"
  226. }
  227. }
  228. // Serialize schema
  229. data, err := json.Marshal(schema)
  230. if err != nil {
  231. return fmt.Errorf("failed to serialize schema: %w", err)
  232. }
  233. // Write schema
  234. err = m.pool.WithClient(func(c *KVClient) error {
  235. return c.Write(key, string(data))
  236. })
  237. if err != nil {
  238. return fmt.Errorf("failed to write schema: %w", err)
  239. }
  240. // Update catalog
  241. if err := m.addToCatalog(schema.Name); err != nil {
  242. // Rollback schema write
  243. m.pool.WithClient(func(c *KVClient) error {
  244. return c.Delete(key)
  245. })
  246. return err
  247. }
  248. // Update cache
  249. m.cache[strings.ToLower(schema.Name)] = schema
  250. m.bumpVersionLocked()
  251. return nil
  252. }
  253. // DropTable drops a table.
  254. func (m *SchemaManager) DropTable(name string) error {
  255. tableLock := m.tableLock(name)
  256. tableLock.Lock()
  257. defer tableLock.Unlock()
  258. m.mu.Lock()
  259. defer m.mu.Unlock()
  260. key := m.schemaKey(name)
  261. // Check if table exists
  262. err := m.pool.WithClient(func(c *KVClient) error {
  263. _, err := c.Read(key)
  264. return err
  265. })
  266. if err != nil {
  267. return fmt.Errorf("table not found: %s", name)
  268. }
  269. // Delete all rows by scanning their actual keys and batch-deleting them, so
  270. // a table drop no longer leaks durable rows.
  271. if err := m.deleteKeysWithPrefix([]byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(name)))); err != nil {
  272. return err
  273. }
  274. // Delete schema
  275. err = m.pool.WithClient(func(c *KVClient) error {
  276. return c.Delete(key)
  277. })
  278. if err != nil {
  279. return fmt.Errorf("failed to delete schema: %w", err)
  280. }
  281. // Delete ROWID state.
  282. m.pool.WithClient(func(c *KVClient) error {
  283. return c.Delete(m.rowIDKey(name))
  284. })
  285. // Update catalog
  286. if err := m.removeFromCatalog(name); err != nil {
  287. return err
  288. }
  289. // Update cache
  290. tableLower := strings.ToLower(name)
  291. delete(m.cache, tableLower)
  292. m.rowIDMu.Lock()
  293. delete(m.rowIDAlloc, tableLower)
  294. m.rowIDMu.Unlock()
  295. m.bumpVersionLocked()
  296. return nil
  297. }
  298. // GetSchema retrieves a table schema.
  299. func (m *SchemaManager) GetSchema(name string) (*Schema, error) {
  300. m.mu.RLock()
  301. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  302. // Clone while still holding the read lock: the cached schema's
  303. // NextRowID field is mutated under the write lock, so cloning outside
  304. // the lock races with that mutation.
  305. cloned := cloneSchema(schema)
  306. m.mu.RUnlock()
  307. return cloned, nil
  308. }
  309. m.mu.RUnlock()
  310. m.mu.Lock()
  311. defer m.mu.Unlock()
  312. // Double-check after acquiring write lock
  313. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  314. return cloneSchema(schema), nil
  315. }
  316. key := m.schemaKey(name)
  317. var data string
  318. err := m.pool.WithClient(func(c *KVClient) error {
  319. var err error
  320. data, err = c.Read(key)
  321. return err
  322. })
  323. if err != nil {
  324. if err == ErrKeyNotFound {
  325. return nil, fmt.Errorf("table not found: %s", name)
  326. }
  327. return nil, err
  328. }
  329. var schema Schema
  330. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  331. return nil, fmt.Errorf("failed to parse schema: %w", err)
  332. }
  333. m.cache[strings.ToLower(name)] = &schema
  334. return cloneSchema(&schema), nil
  335. }
  336. func cloneSchema(schema *Schema) *Schema {
  337. if schema == nil {
  338. return nil
  339. }
  340. cloned := *schema
  341. cloned.Columns = append([]Column(nil), schema.Columns...)
  342. cloned.Checks = append([]CheckConstraint(nil), schema.Checks...)
  343. return &cloned
  344. }
  345. func cloneIndex(index *Index) *Index {
  346. if index == nil {
  347. return nil
  348. }
  349. cloned := *index
  350. cloned.Columns = append([]IndexColumn(nil), index.Columns...)
  351. return &cloned
  352. }
  353. // TableExists checks if a table exists.
  354. func (m *SchemaManager) TableExists(name string) bool {
  355. _, err := m.GetSchema(name)
  356. return err == nil
  357. }
  358. // ListTables returns all table names.
  359. func (m *SchemaManager) ListTables() ([]string, error) {
  360. var data string
  361. err := m.pool.WithClient(func(c *KVClient) error {
  362. var err error
  363. data, err = c.Read(m.catalogKey())
  364. return err
  365. })
  366. if err != nil {
  367. if err == ErrKeyNotFound {
  368. return nil, nil
  369. }
  370. return nil, err
  371. }
  372. var tables []string
  373. if err := json.Unmarshal([]byte(data), &tables); err != nil {
  374. return nil, fmt.Errorf("failed to parse catalog: %w", err)
  375. }
  376. return tables, nil
  377. }
  378. // addToCatalog adds a table to the catalog.
  379. func (m *SchemaManager) addToCatalog(name string) error {
  380. tables, err := m.ListTables()
  381. if err != nil && err != ErrKeyNotFound {
  382. return err
  383. }
  384. // Check if already exists
  385. lowerName := strings.ToLower(name)
  386. for _, t := range tables {
  387. if strings.ToLower(t) == lowerName {
  388. return nil
  389. }
  390. }
  391. tables = append(tables, name)
  392. data, err := json.Marshal(tables)
  393. if err != nil {
  394. return err
  395. }
  396. return m.pool.WithClient(func(c *KVClient) error {
  397. return c.Write(m.catalogKey(), string(data))
  398. })
  399. }
  400. // removeFromCatalog removes a table from the catalog.
  401. func (m *SchemaManager) removeFromCatalog(name string) error {
  402. tables, err := m.ListTables()
  403. if err != nil {
  404. return err
  405. }
  406. lowerName := strings.ToLower(name)
  407. newTables := make([]string, 0, len(tables))
  408. for _, t := range tables {
  409. if strings.ToLower(t) != lowerName {
  410. newTables = append(newTables, t)
  411. }
  412. }
  413. data, err := json.Marshal(newTables)
  414. if err != nil {
  415. return err
  416. }
  417. return m.pool.WithClient(func(c *KVClient) error {
  418. return c.Write(m.catalogKey(), string(data))
  419. })
  420. }
  421. // InvalidateCache clears the cache for a table.
  422. func (m *SchemaManager) InvalidateCache(name string) {
  423. m.mu.Lock()
  424. defer m.mu.Unlock()
  425. tableLower := strings.ToLower(name)
  426. delete(m.cache, tableLower)
  427. m.rowIDMu.Lock()
  428. delete(m.rowIDAlloc, tableLower)
  429. m.rowIDMu.Unlock()
  430. }
  431. // ToAnalyzerTableInfo converts a Schema to analyzer.TableInfo.
  432. func (s *Schema) ToAnalyzerTableInfo() *analyzer.TableInfo {
  433. info := &analyzer.TableInfo{
  434. Name: s.Name,
  435. }
  436. for _, col := range s.Columns {
  437. info.Columns = append(info.Columns, analyzer.ColumnInfo{
  438. Name: col.Name,
  439. Type: analyzer.TypeFromName(col.Type),
  440. Nullable: col.Nullable,
  441. PrimaryKey: col.PrimaryKey,
  442. TableName: s.Name,
  443. Generated: col.GeneratedExpr != "",
  444. })
  445. }
  446. return info
  447. }
  448. // GetColumn returns a column by name.
  449. func (s *Schema) GetColumn(name string) (*Column, bool) {
  450. lowerName := strings.ToLower(name)
  451. for i := range s.Columns {
  452. if strings.ToLower(s.Columns[i].Name) == lowerName {
  453. return &s.Columns[i], true
  454. }
  455. }
  456. return nil, false
  457. }
  458. // GetNextRowID gets and increments the next ROWID for a table. Allocation is
  459. // serialized per table via the table's own allocator so inserts on different
  460. // tables never contend, and no global SchemaManager lock is held across the
  461. // durable derivation scan.
  462. func (m *SchemaManager) GetNextRowID(table string) (int64, error) {
  463. alloc := m.rowIDAllocatorFor(table)
  464. alloc.mu.Lock()
  465. defer alloc.mu.Unlock()
  466. next, err := m.nextRowIDLocked(alloc, table)
  467. if err != nil {
  468. return 0, err
  469. }
  470. alloc.next = next + 1
  471. return next, nil
  472. }
  473. // UpdateMaxRowID updates the next ROWID if the provided value is higher.
  474. func (m *SchemaManager) UpdateMaxRowID(table string, rowid int64) error {
  475. alloc := m.rowIDAllocatorFor(table)
  476. alloc.mu.Lock()
  477. defer alloc.mu.Unlock()
  478. next, err := m.nextRowIDLocked(alloc, table)
  479. if err != nil {
  480. return err
  481. }
  482. if rowid >= next {
  483. alloc.next = rowid + 1
  484. }
  485. return nil
  486. }
  487. // nextRowIDLocked returns the current next ROWID, deriving it from durable rows
  488. // on first use (must hold the per-table allocator lock).
  489. func (m *SchemaManager) nextRowIDLocked(alloc *rowIDAllocator, table string) (int64, error) {
  490. if alloc.init {
  491. return alloc.next, nil
  492. }
  493. next, err := m.deriveNextRowID(table)
  494. if err != nil {
  495. return 0, err
  496. }
  497. if next < 1 {
  498. next = 1
  499. }
  500. alloc.next = next
  501. alloc.init = true
  502. return alloc.next, nil
  503. }
  504. // deriveNextRowID scans durable row keys to recover max(rowid)+1, streaming
  505. // each page through decodeRow instead of materializing every value.
  506. func (m *SchemaManager) deriveNextRowID(table string) (int64, error) {
  507. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  508. var maxRowID int64
  509. err := m.pool.WithClient(func(client *KVClient) (retErr error) {
  510. cursor, err := client.Scan(prefix)
  511. if err != nil {
  512. return err
  513. }
  514. defer func() {
  515. if err := cursor.Close(); retErr == nil {
  516. retErr = err
  517. }
  518. }()
  519. for {
  520. entries, done, err := cursor.Next()
  521. if err != nil {
  522. return err
  523. }
  524. for _, e := range entries {
  525. row, err := decodeRow(e.Value)
  526. if err != nil {
  527. return fmt.Errorf("failed to parse row while deriving ROWID: %w", err)
  528. }
  529. if rowid, ok := valueAsInt64(row["_rowid_"]); ok && rowid > maxRowID {
  530. maxRowID = rowid
  531. }
  532. }
  533. if done {
  534. return nil
  535. }
  536. }
  537. })
  538. if err != nil {
  539. return 0, err
  540. }
  541. return maxRowID + 1, nil
  542. }
  543. // deleteKeysWithPrefix streams the keys with the given prefix one page at a
  544. // time and atomically batch-deletes them, so a bulk operation never leaves
  545. // durable rows behind.
  546. func (m *SchemaManager) deleteKeysWithPrefix(prefix []byte) error {
  547. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  548. cursor, err := client.ScanKeys(prefix)
  549. if err != nil {
  550. return err
  551. }
  552. defer func() {
  553. if err := cursor.Close(); retErr == nil {
  554. retErr = err
  555. }
  556. }()
  557. ops := make([]BatchOp, 0, scanPageSize)
  558. batchBytes := 8
  559. flush := func() error {
  560. if len(ops) == 0 {
  561. return nil
  562. }
  563. if _, err := client.BatchWrite(ops, nil); err != nil {
  564. return err
  565. }
  566. ops = ops[:0]
  567. batchBytes = 8
  568. return nil
  569. }
  570. for {
  571. entries, done, err := cursor.Next()
  572. if err != nil {
  573. return err
  574. }
  575. for _, e := range entries {
  576. opBytes := 12 + len(e.Key)
  577. if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  578. if err := flush(); err != nil {
  579. return err
  580. }
  581. }
  582. ops = append(ops, BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)})
  583. batchBytes += opBytes
  584. }
  585. if done {
  586. return flush()
  587. }
  588. }
  589. })
  590. }
  591. func valueAsInt64(value interface{}) (int64, bool) {
  592. switch v := value.(type) {
  593. case int64:
  594. return v, true
  595. case int:
  596. return int64(v), true
  597. case float64:
  598. return int64(v), true
  599. default:
  600. return 0, false
  601. }
  602. }
  603. // getSchemaLocked retrieves schema (must hold lock).
  604. func (m *SchemaManager) getSchemaLocked(name string) (*Schema, error) {
  605. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  606. return schema, nil
  607. }
  608. key := m.schemaKey(name)
  609. var data string
  610. err := m.pool.WithClient(func(c *KVClient) error {
  611. var err error
  612. data, err = c.Read(key)
  613. return err
  614. })
  615. if err != nil {
  616. if err == ErrKeyNotFound {
  617. return nil, fmt.Errorf("table not found: %s", name)
  618. }
  619. return nil, err
  620. }
  621. var schema Schema
  622. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  623. return nil, fmt.Errorf("failed to parse schema: %w", err)
  624. }
  625. m.cache[strings.ToLower(name)] = &schema
  626. return &schema, nil
  627. }
  628. // saveSchemaLocked saves schema (must hold lock).
  629. func (m *SchemaManager) saveSchemaLocked(schema *Schema) error {
  630. data, err := json.Marshal(schema)
  631. if err != nil {
  632. return fmt.Errorf("failed to serialize schema: %w", err)
  633. }
  634. key := m.schemaKey(schema.Name)
  635. err = m.pool.WithClient(func(c *KVClient) error {
  636. return c.Write(key, string(data))
  637. })
  638. if err != nil {
  639. return fmt.Errorf("failed to write schema: %w", err)
  640. }
  641. m.cache[strings.ToLower(schema.Name)] = schema
  642. return nil
  643. }
  644. // Index management methods
  645. // indexKey returns the key for an index.
  646. func (m *SchemaManager) indexKey(name string) string {
  647. return fmt.Sprintf("%s:index:%s", m.database, strings.ToLower(name))
  648. }
  649. // indexListKey returns the key for the index list.
  650. func (m *SchemaManager) indexListKey() string {
  651. return fmt.Sprintf("%s:indexes", m.database)
  652. }
  653. // CreateIndex creates a new index.
  654. func (m *SchemaManager) CreateIndex(index *Index) error {
  655. m.mu.Lock()
  656. defer m.mu.Unlock()
  657. // Check if index already exists
  658. key := m.indexKey(index.Name)
  659. err := m.pool.WithClient(func(c *KVClient) error {
  660. _, err := c.Read(key)
  661. return err
  662. })
  663. if err == nil {
  664. return fmt.Errorf("index already exists: %s", index.Name)
  665. }
  666. // Verify table exists
  667. if _, err := m.getSchemaLocked(index.Table); err != nil {
  668. return fmt.Errorf("table not found: %s", index.Table)
  669. }
  670. // Save index
  671. index.CreatedAt = time.Now()
  672. data, err := json.Marshal(index)
  673. if err != nil {
  674. return fmt.Errorf("failed to serialize index: %w", err)
  675. }
  676. err = m.pool.WithClient(func(c *KVClient) error {
  677. return c.Write(key, string(data))
  678. })
  679. if err != nil {
  680. return fmt.Errorf("failed to write index: %w", err)
  681. }
  682. // Add to index list
  683. if err := m.addToIndexList(index.Name); err != nil {
  684. return err
  685. }
  686. m.indexCache[strings.ToLower(index.Name)] = cloneIndex(index)
  687. m.bumpVersionLocked()
  688. return nil
  689. }
  690. // DropIndex drops an index.
  691. func (m *SchemaManager) DropIndex(name string) error {
  692. m.mu.Lock()
  693. defer m.mu.Unlock()
  694. key := m.indexKey(name)
  695. err := m.pool.WithClient(func(c *KVClient) error {
  696. return c.Delete(key)
  697. })
  698. if err != nil {
  699. return fmt.Errorf("failed to delete index: %w", err)
  700. }
  701. if err := m.removeFromIndexList(name); err != nil {
  702. return err
  703. }
  704. delete(m.indexCache, strings.ToLower(name))
  705. m.bumpVersionLocked()
  706. return nil
  707. }
  708. // IndexExists checks if an index exists.
  709. func (m *SchemaManager) IndexExists(name string) bool {
  710. m.mu.RLock()
  711. defer m.mu.RUnlock()
  712. return m.indexExistsUnsafe(name)
  713. }
  714. // indexExistsUnsafe reports whether an index definition is durable (m.mu must be
  715. // held).
  716. func (m *SchemaManager) indexExistsUnsafe(name string) bool {
  717. if _, ok := m.indexCache[strings.ToLower(name)]; ok {
  718. return true
  719. }
  720. key := m.indexKey(name)
  721. err := m.pool.WithClient(func(c *KVClient) error {
  722. _, err := c.Read(key)
  723. return err
  724. })
  725. return err == nil
  726. }
  727. // GetIndex retrieves an index by name.
  728. func (m *SchemaManager) GetIndex(name string) (*Index, error) {
  729. m.mu.Lock()
  730. defer m.mu.Unlock()
  731. cacheKey := strings.ToLower(name)
  732. if index, ok := m.indexCache[cacheKey]; ok {
  733. return cloneIndex(index), nil
  734. }
  735. key := m.indexKey(name)
  736. var data string
  737. err := m.pool.WithClient(func(c *KVClient) error {
  738. var err error
  739. data, err = c.Read(key)
  740. return err
  741. })
  742. if err != nil {
  743. if err == ErrKeyNotFound {
  744. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  745. }
  746. return nil, err
  747. }
  748. var index Index
  749. if err := json.Unmarshal([]byte(data), &index); err != nil {
  750. return nil, fmt.Errorf("failed to parse index: %w", err)
  751. }
  752. m.indexCache[cacheKey] = &index
  753. return cloneIndex(&index), nil
  754. }
  755. // ListIndexes returns all index names.
  756. func (m *SchemaManager) ListIndexes() ([]string, error) {
  757. m.mu.Lock()
  758. defer m.mu.Unlock()
  759. return m.indexListUnsafe()
  760. }
  761. // indexListUnsafe returns all index names, loading them from storage on first
  762. // use (m.mu must be held).
  763. func (m *SchemaManager) indexListUnsafe() ([]string, error) {
  764. if m.indexListCached {
  765. return append([]string(nil), m.indexListCache...), nil
  766. }
  767. key := m.indexListKey()
  768. var data string
  769. err := m.pool.WithClient(func(c *KVClient) error {
  770. var err error
  771. data, err = c.Read(key)
  772. return err
  773. })
  774. if err == ErrKeyNotFound {
  775. m.indexListCache = nil
  776. m.indexListCached = true
  777. return []string{}, nil
  778. }
  779. if err != nil {
  780. return nil, err
  781. }
  782. var indexes []string
  783. if err := json.Unmarshal([]byte(data), &indexes); err != nil {
  784. return nil, fmt.Errorf("failed to parse index list: %w", err)
  785. }
  786. m.indexListCache = append([]string(nil), indexes...)
  787. m.indexListCached = true
  788. return indexes, nil
  789. }
  790. // ListTableIndexes returns all indexes for a table.
  791. func (m *SchemaManager) ListTableIndexes(table string) ([]*Index, error) {
  792. indexes, err := m.ListIndexes()
  793. if err != nil {
  794. return nil, err
  795. }
  796. var result []*Index
  797. for _, name := range indexes {
  798. idx, err := m.GetIndex(name)
  799. if err != nil {
  800. // An index dropped concurrently is simply absent; any other error
  801. // (storage/IO or corrupt data) must not be silently swallowed.
  802. if errors.Is(err, ErrIndexNotFound) {
  803. continue
  804. }
  805. return nil, err
  806. }
  807. if strings.EqualFold(idx.Table, table) {
  808. result = append(result, idx)
  809. }
  810. }
  811. return result, nil
  812. }
  813. // addToIndexList adds an index name to the list.
  814. func (m *SchemaManager) addToIndexList(name string) error {
  815. key := m.indexListKey()
  816. var indexes []string
  817. var data string
  818. err := m.pool.WithClient(func(c *KVClient) error {
  819. var err error
  820. data, err = c.Read(key)
  821. return err
  822. })
  823. if err == nil {
  824. json.Unmarshal([]byte(data), &indexes)
  825. }
  826. indexes = append(indexes, name)
  827. newData, _ := json.Marshal(indexes)
  828. err = m.pool.WithClient(func(c *KVClient) error {
  829. return c.Write(key, string(newData))
  830. })
  831. if err == nil {
  832. m.indexListCache = append([]string(nil), indexes...)
  833. m.indexListCached = true
  834. }
  835. return err
  836. }
  837. // removeFromIndexList removes an index name from the list.
  838. func (m *SchemaManager) removeFromIndexList(name string) error {
  839. key := m.indexListKey()
  840. var indexes []string
  841. var data string
  842. err := m.pool.WithClient(func(c *KVClient) error {
  843. var err error
  844. data, err = c.Read(key)
  845. return err
  846. })
  847. if err != nil {
  848. return nil
  849. }
  850. json.Unmarshal([]byte(data), &indexes)
  851. var newIndexes []string
  852. for _, idx := range indexes {
  853. if !strings.EqualFold(idx, name) {
  854. newIndexes = append(newIndexes, idx)
  855. }
  856. }
  857. newData, _ := json.Marshal(newIndexes)
  858. err = m.pool.WithClient(func(c *KVClient) error {
  859. return c.Write(key, string(newData))
  860. })
  861. if err == nil {
  862. m.indexListCache = append([]string(nil), newIndexes...)
  863. m.indexListCached = true
  864. }
  865. return err
  866. }
  867. // AddColumn adds a new column to a table. When unique is non-nil the column's
  868. // UNIQUE index is registered as part of the same statement, and checks are
  869. // persisted with the column in the same schema snapshot.
  870. //
  871. // A constant DEFAULT is materialized into every existing row before the schema
  872. // is published. NOT NULL, UNIQUE, CHECK, and index validation all read
  873. // persisted row values, so a default that only existed in the catalog would be
  874. // invisible to them: a UNIQUE index over the new column could not see the
  875. // duplicate values every existing row shares, a CHECK predicate could not see a
  876. // violating default, and an UPDATE to NULL would look non-null. The rewrite is
  877. // the same shape as DropColumn: the table's exclusive gate keeps point writes
  878. // and CAS commits out while rows are scanned, validated, and rewritten, the
  879. // transform is idempotent, and the schema is saved last so an interrupted
  880. // operation resumes against the old schema.
  881. //
  882. // The gate also makes the validation authoritative: NOT NULL without a default,
  883. // duplicate UNIQUE values, and rows a new CHECK predicate rejects are rejected
  884. // before any row is touched, and the column, its checks, and its index are
  885. // published together in one batch write, so a crash cannot leave the column
  886. // visible without the constraints that validate it. A nullable column without a
  887. // default needs no rewrite because a missing row field already reads as NULL.
  888. // Like DROP COLUMN, the statement holds the catalog lock for the whole scan, so
  889. // the cost is one table scan per validating constraint plus a table-wide stall
  890. // per ADD COLUMN that has a default, an index, or a check.
  891. func (m *SchemaManager) AddColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
  892. tableLock := m.tableLock(table)
  893. tableLock.Lock()
  894. defer tableLock.Unlock()
  895. m.mu.Lock()
  896. defer m.mu.Unlock()
  897. schema, err := m.getSchemaUnsafe(table)
  898. if err != nil {
  899. return err
  900. }
  901. if err := checkColumnAbsent(schema, column.Name); err != nil {
  902. return err
  903. }
  904. if unique != nil {
  905. if !strings.EqualFold(unique.Table, schema.Name) {
  906. return fmt.Errorf("index %s belongs to table %s, not %s", unique.Name, unique.Table, schema.Name)
  907. }
  908. if m.indexExistsUnsafe(unique.Name) {
  909. return fmt.Errorf("index already exists: %s", unique.Name)
  910. }
  911. if err := m.validateAddColumnUnique(table, column, unique); err != nil {
  912. return err
  913. }
  914. }
  915. if err := m.validateAddColumnChecks(table, column, checks); err != nil {
  916. return err
  917. }
  918. // A required column with nothing to write into rows that already exist
  919. // cannot be declared on a populated table.
  920. if !column.Nullable && column.Default == nil {
  921. hasRows, err := m.tableHasRows(table)
  922. if err != nil {
  923. return err
  924. }
  925. if hasRows {
  926. return &NotNullViolationError{Table: schema.Name, Column: column.Name}
  927. }
  928. }
  929. if column.Default != nil {
  930. if err := m.rewriteRows(table, func(row Row) bool {
  931. if _, ok := rowColumnValue(row, column.Name); ok {
  932. return false
  933. }
  934. row[column.Name] = column.Default
  935. return true
  936. }); err != nil {
  937. return err
  938. }
  939. }
  940. return m.publishAddedColumn(schema, column, checks, unique)
  941. }
  942. // publishAddedColumn publishes a column, its checks, and, when present, its
  943. // unique index in one atomic batch write (m.mu must be held): the schema
  944. // snapshot, the index definition, and the index list always become durable
  945. // together or not at all.
  946. func (m *SchemaManager) publishAddedColumn(schema *Schema, column Column, checks []CheckConstraint, unique *Index) error {
  947. // Publish a fresh snapshot instead of mutating readers' shared pointer.
  948. schema = cloneSchema(schema)
  949. schema.Columns = append(schema.Columns, column)
  950. schema.Checks = append(schema.Checks, checks...)
  951. schemaData, err := json.Marshal(schema)
  952. if err != nil {
  953. return fmt.Errorf("failed to serialize schema: %w", err)
  954. }
  955. ops := []BatchOp{{Op: batchPut, Key: []byte(m.schemaKey(schema.Name)), Value: schemaData}}
  956. var indexList []string
  957. if unique != nil {
  958. unique.CreatedAt = time.Now()
  959. indexData, err := json.Marshal(unique)
  960. if err != nil {
  961. return fmt.Errorf("failed to serialize index: %w", err)
  962. }
  963. indexList, err = m.indexListUnsafe()
  964. if err != nil {
  965. return err
  966. }
  967. indexList = append(indexList, unique.Name)
  968. listData, err := json.Marshal(indexList)
  969. if err != nil {
  970. return fmt.Errorf("failed to serialize index list: %w", err)
  971. }
  972. ops = append(ops,
  973. BatchOp{Op: batchPut, Key: []byte(m.indexKey(unique.Name)), Value: indexData},
  974. BatchOp{Op: batchPut, Key: []byte(m.indexListKey()), Value: listData},
  975. )
  976. }
  977. if err := m.pool.WithClient(func(c *KVClient) error {
  978. _, err := c.BatchWrite(ops, nil)
  979. return err
  980. }); err != nil {
  981. return fmt.Errorf("failed to publish column %s: %w", column.Name, err)
  982. }
  983. // Publish in-memory state only after the durable write.
  984. m.cache[strings.ToLower(schema.Name)] = schema
  985. if unique != nil {
  986. m.indexCache[strings.ToLower(unique.Name)] = cloneIndex(unique)
  987. m.indexListCache = append([]string(nil), indexList...)
  988. m.indexListCached = true
  989. }
  990. m.bumpVersionLocked()
  991. return nil
  992. }
  993. func checkColumnAbsent(schema *Schema, name string) error {
  994. for _, col := range schema.Columns {
  995. if strings.EqualFold(col.Name, name) {
  996. return fmt.Errorf("column already exists: %s", name)
  997. }
  998. }
  999. return nil
  1000. }
  1001. // DropColumn removes a column from a table.
  1002. func (m *SchemaManager) DropColumn(table, columnName string) error {
  1003. tableLock := m.tableLock(table)
  1004. tableLock.Lock()
  1005. defer tableLock.Unlock()
  1006. m.mu.Lock()
  1007. defer m.mu.Unlock()
  1008. schema, err := m.getSchemaUnsafe(table)
  1009. if err != nil {
  1010. return err
  1011. }
  1012. // Cannot drop primary key column
  1013. if strings.EqualFold(schema.PrimaryKey, columnName) {
  1014. return fmt.Errorf("cannot drop primary key column: %s", columnName)
  1015. }
  1016. // Find and remove column
  1017. newColumns := make([]Column, 0, len(schema.Columns)-1)
  1018. found := false
  1019. for _, col := range schema.Columns {
  1020. if strings.EqualFold(col.Name, columnName) {
  1021. found = true
  1022. continue
  1023. }
  1024. newColumns = append(newColumns, col)
  1025. }
  1026. if !found {
  1027. return fmt.Errorf("column not found: %s", columnName)
  1028. }
  1029. schema = cloneSchema(schema)
  1030. schema.Columns = newColumns
  1031. if err := m.rewriteRows(table, func(row Row) bool {
  1032. for key := range row {
  1033. if strings.EqualFold(key, columnName) {
  1034. delete(row, key)
  1035. return true
  1036. }
  1037. }
  1038. return false
  1039. }); err != nil {
  1040. return err
  1041. }
  1042. // Update schema
  1043. return m.updateSchemaUnsafe(schema)
  1044. }
  1045. // RenameTable renames a table.
  1046. func (m *SchemaManager) RenameTable(oldName, newName string) error {
  1047. m.mu.Lock()
  1048. defer m.mu.Unlock()
  1049. // Check if old table exists
  1050. schema, err := m.getSchemaUnsafe(oldName)
  1051. if err != nil {
  1052. return err
  1053. }
  1054. // Check if new table name already exists
  1055. _, err = m.getSchemaUnsafe(newName)
  1056. if err == nil {
  1057. return fmt.Errorf("table already exists: %s", newName)
  1058. }
  1059. schema = cloneSchema(schema)
  1060. // Update schema name
  1061. schema.Name = newName
  1062. // Move durable rows from the old table's data prefix to the new one. Without
  1063. // this, ALTER TABLE ... RENAME leaves every row under the old key and the
  1064. // renamed table appears empty.
  1065. if err := m.renameDataKeys(oldName, newName); err != nil {
  1066. return err
  1067. }
  1068. // Repoint indexes that belonged to the old table.
  1069. if err := m.repointIndexesLocked(oldName, newName); err != nil {
  1070. return err
  1071. }
  1072. // Build the replacement catalog before switching schema names. The schema,
  1073. // catalog, and stale ROWID state change in one batch, so a crash cannot
  1074. // leave neither table name addressable after all row chunks have moved.
  1075. tables, err := m.ListTables()
  1076. if err != nil {
  1077. return err
  1078. }
  1079. foundOld := false
  1080. for i, name := range tables {
  1081. if strings.EqualFold(name, oldName) {
  1082. tables[i] = newName
  1083. foundOld = true
  1084. break
  1085. }
  1086. }
  1087. if !foundOld {
  1088. return fmt.Errorf("table not found in catalog: %s", oldName)
  1089. }
  1090. catalogData, err := json.Marshal(tables)
  1091. if err != nil {
  1092. return err
  1093. }
  1094. schemaData, err := json.Marshal(schema)
  1095. if err != nil {
  1096. return err
  1097. }
  1098. oldKey := m.schemaKey(oldName)
  1099. newKey := m.schemaKey(newName)
  1100. err = m.pool.WithClient(func(c *KVClient) error {
  1101. _, err := c.BatchWrite([]BatchOp{
  1102. {Op: batchPut, Key: []byte(newKey), Value: schemaData},
  1103. {Op: batchDelete, Key: []byte(oldKey)},
  1104. {Op: batchDelete, Key: []byte(m.rowIDKey(oldName))},
  1105. {Op: batchPut, Key: []byte(m.catalogKey()), Value: catalogData},
  1106. }, nil)
  1107. return err
  1108. })
  1109. if err != nil {
  1110. return err
  1111. }
  1112. // Update cache
  1113. oldLower := strings.ToLower(oldName)
  1114. newLower := strings.ToLower(newName)
  1115. delete(m.cache, oldLower)
  1116. m.rowIDMu.Lock()
  1117. delete(m.rowIDAlloc, oldLower)
  1118. m.rowIDMu.Unlock()
  1119. // Update cache
  1120. m.cache[newLower] = schema
  1121. m.bumpVersionLocked()
  1122. return nil
  1123. }
  1124. // renameDataKeys moves every durable row of a table from the old name's data
  1125. // prefix to the new name's prefix in a single atomic batch per page.
  1126. func (m *SchemaManager) renameDataKeys(oldName, newName string) error {
  1127. oldPrefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(oldName)))
  1128. newPrefix := fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(newName))
  1129. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  1130. cursor, err := client.Scan(oldPrefix)
  1131. if err != nil {
  1132. return err
  1133. }
  1134. defer func() {
  1135. if cerr := cursor.Close(); retErr == nil {
  1136. retErr = cerr
  1137. }
  1138. }()
  1139. ops := make([]BatchOp, 0, scanPageSize*2)
  1140. batchBytes := 8
  1141. flush := func() error {
  1142. if len(ops) == 0 {
  1143. return nil
  1144. }
  1145. if _, err := client.BatchWrite(ops, nil); err != nil {
  1146. return err
  1147. }
  1148. ops = ops[:0]
  1149. batchBytes = 8
  1150. return nil
  1151. }
  1152. for {
  1153. entries, done, err := cursor.Next()
  1154. if err != nil {
  1155. return err
  1156. }
  1157. for _, e := range entries {
  1158. suffix := string(e.Key[len(oldPrefix):])
  1159. newKey := []byte(newPrefix + suffix)
  1160. opBytes := 24 + len(newKey) + len(e.Value) + len(e.Key)
  1161. if len(ops)+2 > maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  1162. if err := flush(); err != nil {
  1163. return err
  1164. }
  1165. }
  1166. ops = append(ops,
  1167. BatchOp{Op: batchPut, Key: newKey, Value: e.Value},
  1168. BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)},
  1169. )
  1170. batchBytes += opBytes
  1171. }
  1172. if done {
  1173. return flush()
  1174. }
  1175. }
  1176. })
  1177. }
  1178. // updateIndexesOfTableLocked applies mutate to the durable definition of every
  1179. // index on table and persists the definitions it changed. mutate reports
  1180. // whether it modified the definition; unmodified indexes are left untouched.
  1181. // The changed definitions are written in one batch, so a table's indexes never
  1182. // split between the old and new shape. The caller holds m.mu; it reads the
  1183. // durable index list directly rather than calling the self-locking ListIndexes
  1184. // helper.
  1185. func (m *SchemaManager) updateIndexesOfTableLocked(table string, mutate func(*Index) bool) error {
  1186. var names []string
  1187. if m.indexListCached {
  1188. names = append([]string(nil), m.indexListCache...)
  1189. } else {
  1190. var data string
  1191. err := m.pool.WithClient(func(c *KVClient) error {
  1192. var rerr error
  1193. data, rerr = c.Read(m.indexListKey())
  1194. return rerr
  1195. })
  1196. if err == ErrKeyNotFound {
  1197. return nil
  1198. }
  1199. if err != nil {
  1200. return err
  1201. }
  1202. if err := json.Unmarshal([]byte(data), &names); err != nil {
  1203. return err
  1204. }
  1205. }
  1206. type indexUpdate struct {
  1207. index Index
  1208. key string
  1209. value []byte
  1210. }
  1211. var updates []indexUpdate
  1212. for _, name := range names {
  1213. key := m.indexKey(name)
  1214. var data string
  1215. err := m.pool.WithClient(func(c *KVClient) error {
  1216. var rerr error
  1217. data, rerr = c.Read(key)
  1218. return rerr
  1219. })
  1220. if err != nil {
  1221. if err == ErrKeyNotFound {
  1222. continue
  1223. }
  1224. return err
  1225. }
  1226. var idx Index
  1227. if err := json.Unmarshal([]byte(data), &idx); err != nil {
  1228. return err
  1229. }
  1230. if !strings.EqualFold(idx.Table, table) {
  1231. continue
  1232. }
  1233. if !mutate(&idx) {
  1234. continue
  1235. }
  1236. updated, err := json.Marshal(&idx)
  1237. if err != nil {
  1238. return err
  1239. }
  1240. updates = append(updates, indexUpdate{index: idx, key: key, value: updated})
  1241. }
  1242. if len(updates) == 0 {
  1243. return nil
  1244. }
  1245. ops := make([]BatchOp, 0, len(updates))
  1246. for _, update := range updates {
  1247. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(update.key), Value: update.value})
  1248. }
  1249. if err := m.pool.WithClient(func(c *KVClient) error {
  1250. _, err := c.BatchWrite(ops, nil)
  1251. return err
  1252. }); err != nil {
  1253. return fmt.Errorf("update index definitions for table %s: %w", table, err)
  1254. }
  1255. for _, update := range updates {
  1256. if cached, ok := m.indexCache[strings.ToLower(update.index.Name)]; ok {
  1257. *cached = update.index
  1258. }
  1259. }
  1260. return nil
  1261. }
  1262. // repointIndexesLocked updates indexes whose table was renamed.
  1263. func (m *SchemaManager) repointIndexesLocked(oldName, newName string) error {
  1264. return m.updateIndexesOfTableLocked(oldName, func(idx *Index) bool {
  1265. idx.Table = newName
  1266. return true
  1267. })
  1268. }
  1269. // renameIndexColumnsLocked rewrites the stored definition of every index on
  1270. // table so a reference to oldName becomes newName. Index names stay unchanged:
  1271. // they are independent identifiers that SQLite does not derive from column
  1272. // names, and callers may have chosen them explicitly. A rename to an existing
  1273. // column name is rejected before this runs. Expression columns keep their SQL
  1274. // text; callers must reject a rename that such an expression depends on,
  1275. // because the text is not rewritten.
  1276. func (m *SchemaManager) renameIndexColumnsLocked(table, oldName, newName string) error {
  1277. return m.updateIndexesOfTableLocked(table, func(idx *Index) bool {
  1278. changed := false
  1279. for i := range idx.Columns {
  1280. // Expression columns persist SQL text, not a column reference; their
  1281. // text is left for the caller to drop and recreate the index.
  1282. if idx.Columns[i].Expression == "" && strings.EqualFold(idx.Columns[i].Name, oldName) {
  1283. idx.Columns[i].Name = newName
  1284. changed = true
  1285. }
  1286. }
  1287. return changed
  1288. })
  1289. }
  1290. // RenameColumn renames a column in a table. Column references in index
  1291. // definitions are rewritten; expression-index text is not, so a SQL caller must
  1292. // reject a rename that an expression depends on.
  1293. func (m *SchemaManager) RenameColumn(table, oldName, newName string) error {
  1294. tableLock := m.tableLock(table)
  1295. tableLock.Lock()
  1296. defer tableLock.Unlock()
  1297. m.mu.Lock()
  1298. defer m.mu.Unlock()
  1299. schema, err := m.getSchemaUnsafe(table)
  1300. if err != nil {
  1301. return err
  1302. }
  1303. // Check if new column name already exists
  1304. for _, col := range schema.Columns {
  1305. if strings.EqualFold(col.Name, newName) {
  1306. return fmt.Errorf("column already exists: %s", newName)
  1307. }
  1308. }
  1309. schema = cloneSchema(schema)
  1310. // Find and rename column
  1311. found := false
  1312. for i, col := range schema.Columns {
  1313. if strings.EqualFold(col.Name, oldName) {
  1314. schema.Columns[i].Name = newName
  1315. found = true
  1316. // Update primary key reference if needed
  1317. if strings.EqualFold(schema.PrimaryKey, oldName) {
  1318. schema.PrimaryKey = newName
  1319. }
  1320. break
  1321. }
  1322. }
  1323. if !found {
  1324. return fmt.Errorf("column not found: %s", oldName)
  1325. }
  1326. if err := m.rewriteRows(table, func(row Row) bool {
  1327. for key, value := range row {
  1328. if strings.EqualFold(key, oldName) {
  1329. delete(row, key)
  1330. row[newName] = value
  1331. return true
  1332. }
  1333. }
  1334. return false
  1335. }); err != nil {
  1336. return err
  1337. }
  1338. // Publish the renamed schema before the index definitions: reads of the new
  1339. // column name resolve against rewritten rows as soon as the schema lands.
  1340. // Index DDL is not transactional (as elsewhere in this engine), so a failure
  1341. // here leaves stale index metadata for the caller to drop and recreate.
  1342. if err := m.updateSchemaUnsafe(schema); err != nil {
  1343. return err
  1344. }
  1345. // Index metadata drives secondary-index values, UNIQUE enforcement,
  1346. // ON CONFLICT target resolution, and catalog introspection, so it must name
  1347. // the renamed column before any later statement uses the table.
  1348. return m.renameIndexColumnsLocked(table, oldName, newName)
  1349. }
  1350. // rewriteRows applies an idempotent name-keyed row transformation in bounded
  1351. // batches. Column DDL updates the schema only after all rows are rewritten, so
  1352. // an interrupted operation can safely resume against the old schema.
  1353. func (m *SchemaManager) rewriteRows(table string, transform func(Row) bool) error {
  1354. return m.applyRows(table, func(row Row) (bool, error) {
  1355. return transform(row), nil
  1356. })
  1357. }
  1358. // applyRows scans a table once and applies transform to every row. Rows for
  1359. // which transform reports a change are re-encoded and written in bounded
  1360. // batches; an error from transform aborts the scan. Callers hold the table's
  1361. // exclusive gate, so no writer can interleave between the scan and the schema
  1362. // update that follows it. Aborting after a batch was flushed is safe only
  1363. // because the DDL transforms are idempotent.
  1364. func (m *SchemaManager) applyRows(table string, transform func(Row) (bool, error)) error {
  1365. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  1366. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  1367. cursor, err := client.Scan(prefix)
  1368. if err != nil {
  1369. return err
  1370. }
  1371. defer func() {
  1372. if cerr := cursor.Close(); retErr == nil {
  1373. retErr = cerr
  1374. }
  1375. }()
  1376. ops := make([]BatchOp, 0, scanPageSize)
  1377. batchBytes := 8
  1378. flush := func() error {
  1379. if len(ops) == 0 {
  1380. return nil
  1381. }
  1382. if _, err := client.BatchWrite(ops, nil); err != nil {
  1383. return err
  1384. }
  1385. ops = ops[:0]
  1386. batchBytes = 8
  1387. return nil
  1388. }
  1389. for {
  1390. entries, done, err := cursor.Next()
  1391. if err != nil {
  1392. return err
  1393. }
  1394. for _, entry := range entries {
  1395. row, err := decodeRow(entry.Value)
  1396. if err != nil {
  1397. return err
  1398. }
  1399. changed, err := transform(row)
  1400. if err != nil {
  1401. return err
  1402. }
  1403. if !changed {
  1404. continue
  1405. }
  1406. value, err := encodeRow(row)
  1407. if err != nil {
  1408. return err
  1409. }
  1410. opBytes := 12 + len(entry.Key) + len(value)
  1411. if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  1412. if err := flush(); err != nil {
  1413. return err
  1414. }
  1415. }
  1416. ops = append(ops, BatchOp{Op: batchPut, Key: append([]byte(nil), entry.Key...), Value: value})
  1417. batchBytes += opBytes
  1418. }
  1419. if done {
  1420. return flush()
  1421. }
  1422. }
  1423. })
  1424. }
  1425. // validateAddColumnUnique rejects an ADD COLUMN ... UNIQUE whose backfilled
  1426. // values would duplicate across existing rows. It reads rows as they will look
  1427. // after the rewrite (a row that already carries the field from an interrupted
  1428. // earlier attempt keeps its value) and mirrors ValidateUniqueIndex for a single
  1429. // plain column: NULLs do not collide. The caller holds the table's exclusive
  1430. // gate, so no writer can insert a conflicting row between this check and the
  1431. // publication of the index.
  1432. func (m *SchemaManager) validateAddColumnUnique(table string, column Column, index *Index) error {
  1433. seen := make(map[string]struct{})
  1434. return m.applyRows(table, func(row Row) (bool, error) {
  1435. value, ok := rowColumnValue(row, column.Name)
  1436. if !ok {
  1437. value = column.Default
  1438. }
  1439. if value == nil {
  1440. return false, nil
  1441. }
  1442. encoded := encodeUniqueScalar(value)
  1443. if _, duplicate := seen[encoded]; duplicate {
  1444. return false, &UniqueViolationError{Constraint: index.Name}
  1445. }
  1446. seen[encoded] = struct{}{}
  1447. return false, nil
  1448. })
  1449. }
  1450. // validateAddColumnChecks rejects an ADD COLUMN ... CHECK whose predicate is
  1451. // already false for an existing row. Like validateAddColumnUnique it reads rows
  1452. // as they will look after the constant-default backfill, and it runs before that
  1453. // rewrite so a rejected DDL leaves no row carrying the new field. The caller
  1454. // holds the table's exclusive gate, so no writer can insert a violating row
  1455. // between this validation and the publication of the check.
  1456. func (m *SchemaManager) validateAddColumnChecks(table string, column Column, checks []CheckConstraint) error {
  1457. if len(checks) == 0 {
  1458. return nil
  1459. }
  1460. eval := m.checkEvaluator()
  1461. if eval == nil {
  1462. return fmt.Errorf("cannot validate CHECK constraints on %s: no CHECK evaluator installed", table)
  1463. }
  1464. return m.applyRows(table, func(row Row) (bool, error) {
  1465. if _, ok := rowColumnValue(row, column.Name); !ok && column.Default != nil {
  1466. row[column.Name] = column.Default
  1467. }
  1468. for _, check := range checks {
  1469. holds, err := eval(check.Expression, row)
  1470. if err != nil {
  1471. return false, fmt.Errorf("evaluate CHECK constraint %s on %s: %w", check.Label(), table, err)
  1472. }
  1473. if !holds {
  1474. return false, &CheckViolationError{Table: table, Constraint: check.Label()}
  1475. }
  1476. }
  1477. return false, nil
  1478. })
  1479. }
  1480. // tableHasRows reports whether a table holds at least one durable row.
  1481. func (m *SchemaManager) tableHasRows(table string) (bool, error) {
  1482. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  1483. found := false
  1484. err := m.pool.WithClient(func(client *KVClient) (retErr error) {
  1485. cursor, err := client.ScanWithLimit(prefix, 1)
  1486. if err != nil {
  1487. return err
  1488. }
  1489. defer func() {
  1490. if cerr := cursor.Close(); retErr == nil {
  1491. retErr = cerr
  1492. }
  1493. }()
  1494. entries, _, err := cursor.Next()
  1495. if err != nil {
  1496. return err
  1497. }
  1498. found = len(entries) > 0
  1499. return nil
  1500. })
  1501. return found, err
  1502. }
  1503. // getSchemaUnsafe gets a schema without locking (internal use).
  1504. func (m *SchemaManager) getSchemaUnsafe(table string) (*Schema, error) {
  1505. tableLower := strings.ToLower(table)
  1506. // Check cache
  1507. if schema, ok := m.cache[tableLower]; ok {
  1508. return schema, nil
  1509. }
  1510. // Read from storage
  1511. key := m.schemaKey(table)
  1512. var data string
  1513. err := m.pool.WithClient(func(c *KVClient) error {
  1514. var err error
  1515. data, err = c.Read(key)
  1516. return err
  1517. })
  1518. if err != nil {
  1519. return nil, fmt.Errorf("table not found: %s", table)
  1520. }
  1521. var schema Schema
  1522. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  1523. return nil, err
  1524. }
  1525. m.cache[tableLower] = &schema
  1526. return &schema, nil
  1527. }
  1528. // updateSchemaUnsafe updates a schema without locking (internal use).
  1529. func (m *SchemaManager) updateSchemaUnsafe(schema *Schema) error {
  1530. key := m.schemaKey(schema.Name)
  1531. data, _ := json.Marshal(schema)
  1532. err := m.pool.WithClient(func(c *KVClient) error {
  1533. return c.Write(key, string(data))
  1534. })
  1535. if err != nil {
  1536. return err
  1537. }
  1538. // Update cache
  1539. m.cache[strings.ToLower(schema.Name)] = schema
  1540. m.bumpVersionLocked()
  1541. return nil
  1542. }