schema.go 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728
  1. package storage
  2. import (
  3. "errors"
  4. "fmt"
  5. "strings"
  6. "sync"
  7. "time"
  8. "github.com/goccy/go-json"
  9. "github.com/danfragoso/pizzasql-next/pkg/analyzer"
  10. )
  11. // ErrIndexNotFound is returned by GetIndex when an index is absent. It is
  12. // distinct from a storage/IO error so callers like ListTableIndexes can skip a
  13. // concurrently-dropped index without swallowing real read failures.
  14. var ErrIndexNotFound = errors.New("index not found")
  15. // Schema represents a table schema.
  16. type Schema struct {
  17. Name string `json:"name"`
  18. Columns []Column `json:"columns"`
  19. PrimaryKey string `json:"primary_key"`
  20. CreatedAt time.Time `json:"created_at"`
  21. NextRowID int64 `json:"next_rowid"`
  22. AutoIncrement bool `json:"autoincrement"`
  23. }
  24. // Column represents a column definition.
  25. type Column struct {
  26. Name string `json:"name"`
  27. Type string `json:"type"`
  28. Nullable bool `json:"nullable"`
  29. // Default holds the evaluated constant DEFAULT. DefaultExpr holds the SQL
  30. // text of a non-constant DEFAULT (CURRENT_TIMESTAMP and the other
  31. // clock-reading forms) so it is evaluated on every write instead of being
  32. // frozen at DDL time. Only one of the two is set; DefaultExpr takes
  33. // precedence if a hand-edited catalog sets both.
  34. Default interface{} `json:"default,omitempty"`
  35. DefaultExpr string `json:"default_expr,omitempty"`
  36. PrimaryKey bool `json:"primary_key"`
  37. // GeneratedExpr holds the SQL text of a GENERATED ALWAYS AS (expr) column.
  38. // GeneratedStored reports whether the value is materialized on write
  39. // (STORED, the only form this engine persists) versus computed on read.
  40. GeneratedExpr string `json:"generated_expr,omitempty"`
  41. GeneratedStored bool `json:"generated_stored,omitempty"`
  42. }
  43. // DefaultLiteral returns the catalog representation of the column's default:
  44. // the evaluated constant value, or the SQL text of a non-constant default.
  45. // DefaultExpr takes precedence when a hand-edited catalog sets both.
  46. func (c Column) DefaultLiteral() interface{} {
  47. if c.DefaultExpr != "" {
  48. return c.DefaultExpr
  49. }
  50. if c.Default != nil {
  51. return c.Default
  52. }
  53. return nil
  54. }
  55. // Index represents an index definition.
  56. type Index struct {
  57. Name string `json:"name"`
  58. Table string `json:"table"`
  59. Columns []IndexColumn `json:"columns"`
  60. Unique bool `json:"unique"`
  61. CreatedAt time.Time `json:"created_at"`
  62. // OnConflict is the default conflict resolution declared for this index via
  63. // a UNIQUE(...) ON CONFLICT clause. Empty means the SQLite default (ABORT).
  64. OnConflict string `json:"on_conflict,omitempty"`
  65. }
  66. // IndexColumn represents a column in an index. Expression is set for expression
  67. // indexes (e.g. lower(email)); a plain column index leaves it empty and uses
  68. // Name.
  69. type IndexColumn struct {
  70. Name string `json:"name"`
  71. Desc bool `json:"desc"`
  72. Expression string `json:"expression,omitempty"`
  73. }
  74. // rowIDAllocator owns the next-ROWID state for a single table. It is a separate
  75. // mutex per table so allocating a ROWID on one table never serializes against
  76. // another table, and never contends with the SchemaManager catalog lock.
  77. type rowIDAllocator struct {
  78. mu sync.Mutex
  79. next int64
  80. init bool
  81. }
  82. // SchemaManager manages table schemas.
  83. type SchemaManager struct {
  84. pool *KVPool
  85. database string
  86. cache map[string]*Schema
  87. indexCache map[string]*Index
  88. indexListCache []string
  89. indexListCached bool
  90. version uint64
  91. mu sync.RWMutex
  92. tableLocksMu sync.Mutex
  93. tableLocks map[string]*sync.RWMutex
  94. rowIDMu sync.Mutex
  95. rowIDAlloc map[string]*rowIDAllocator
  96. // defaultEval evaluates a persisted non-constant DEFAULT expression. The
  97. // SQL executor registers one so storage can backfill ADD COLUMN rows and
  98. // complete row images without importing the parser or executor.
  99. defaultEvalMu sync.RWMutex
  100. defaultEval DefaultEvaluator
  101. }
  102. // DefaultEvaluator evaluates a persisted non-constant DEFAULT expression, such
  103. // as CURRENT_TIMESTAMP, to the value a row receives. Expression defaults are
  104. // never frozen at DDL time: each write reads the clock again.
  105. type DefaultEvaluator func(expression string) (interface{}, error)
  106. // SetDefaultEvaluator installs the evaluator used for expression defaults.
  107. func (m *SchemaManager) SetDefaultEvaluator(eval DefaultEvaluator) {
  108. m.defaultEvalMu.Lock()
  109. m.defaultEval = eval
  110. m.defaultEvalMu.Unlock()
  111. }
  112. func (m *SchemaManager) defaultEvaluator() DefaultEvaluator {
  113. m.defaultEvalMu.RLock()
  114. defer m.defaultEvalMu.RUnlock()
  115. return m.defaultEval
  116. }
  117. // BeginTransaction is retained for API compatibility. Buffered per-session
  118. // transactions no longer take a database-wide transaction lock; staged writes
  119. // are validated and committed atomically with CompareBatchWrite instead.
  120. func (m *SchemaManager) BeginTransaction() {}
  121. // EndTransaction is retained for API compatibility.
  122. func (m *SchemaManager) EndTransaction() {}
  123. // LockStatement is retained for API compatibility. Statement execution is now
  124. // serialized through per-table locks and optimistic validation, so no global
  125. // statement lock is required.
  126. func (m *SchemaManager) LockStatement() {}
  127. // UnlockStatement is retained for API compatibility.
  128. func (m *SchemaManager) UnlockStatement() {}
  129. // NewSchemaManager creates a new schema manager.
  130. func NewSchemaManager(pool *KVPool, database string) *SchemaManager {
  131. return &SchemaManager{
  132. pool: pool,
  133. database: database,
  134. cache: make(map[string]*Schema),
  135. indexCache: make(map[string]*Index),
  136. tableLocks: make(map[string]*sync.RWMutex),
  137. rowIDAlloc: make(map[string]*rowIDAllocator),
  138. }
  139. }
  140. // rowIDAllocatorFor returns (creating if needed) the per-table ROWID allocator.
  141. func (m *SchemaManager) rowIDAllocatorFor(table string) *rowIDAllocator {
  142. key := strings.ToLower(table)
  143. m.rowIDMu.Lock()
  144. a, ok := m.rowIDAlloc[key]
  145. if !ok {
  146. a = &rowIDAllocator{}
  147. m.rowIDAlloc[key] = a
  148. }
  149. m.rowIDMu.Unlock()
  150. return a
  151. }
  152. func (m *SchemaManager) tableLock(table string) *sync.RWMutex {
  153. key := strings.ToLower(table)
  154. m.tableLocksMu.Lock()
  155. lock, ok := m.tableLocks[key]
  156. if !ok {
  157. lock = &sync.RWMutex{}
  158. m.tableLocks[key] = lock
  159. }
  160. m.tableLocksMu.Unlock()
  161. return lock
  162. }
  163. // GetDatabaseName returns the database name.
  164. func (m *SchemaManager) GetDatabaseName() string {
  165. return m.database
  166. }
  167. // GetPool returns the KV pool.
  168. func (m *SchemaManager) GetPool() *KVPool {
  169. return m.pool
  170. }
  171. // Version returns the in-process schema catalog version. It is incremented for
  172. // schema/index definition changes so cached executors can resync their analyzer
  173. // catalogs without scanning storage on every query.
  174. func (m *SchemaManager) Version() uint64 {
  175. m.mu.RLock()
  176. defer m.mu.RUnlock()
  177. return m.version
  178. }
  179. func (m *SchemaManager) bumpVersionLocked() {
  180. m.version++
  181. }
  182. // schemaKey returns the key for a table schema.
  183. func (m *SchemaManager) schemaKey(table string) string {
  184. return fmt.Sprintf("%s:_schema:%s", m.database, strings.ToLower(table))
  185. }
  186. // catalogKey returns the key for the table catalog.
  187. func (m *SchemaManager) catalogKey() string {
  188. return fmt.Sprintf("%s:_sys:tables", m.database)
  189. }
  190. // rowIDKey returns the key for a table's next ROWID counter.
  191. func (m *SchemaManager) rowIDKey(table string) string {
  192. return fmt.Sprintf("%s:_sys:rowid:%s", m.database, strings.ToLower(table))
  193. }
  194. // CreateTable creates a new table.
  195. func (m *SchemaManager) CreateTable(schema *Schema) error {
  196. m.mu.Lock()
  197. defer m.mu.Unlock()
  198. // Check if table already exists
  199. key := m.schemaKey(schema.Name)
  200. err := m.pool.WithClient(func(c *KVClient) error {
  201. _, err := c.Read(key)
  202. return err
  203. })
  204. if err == nil {
  205. return fmt.Errorf("table already exists: %s", schema.Name)
  206. }
  207. // Keep the cached schema private so callers cannot mutate a published
  208. // catalog snapshot after this operation returns.
  209. schema = cloneSchema(schema)
  210. schema.CreatedAt = time.Now()
  211. // Determine primary key if not set
  212. if schema.PrimaryKey == "" {
  213. for _, col := range schema.Columns {
  214. if col.PrimaryKey {
  215. schema.PrimaryKey = col.Name
  216. break
  217. }
  218. }
  219. // No explicit primary key declared — use synthetic _rowid_ so user
  220. // columns remain unconstrained and can hold duplicate or NULL values.
  221. if schema.PrimaryKey == "" {
  222. schema.PrimaryKey = "_rowid_"
  223. }
  224. }
  225. // Serialize schema
  226. data, err := json.Marshal(schema)
  227. if err != nil {
  228. return fmt.Errorf("failed to serialize schema: %w", err)
  229. }
  230. // Write schema
  231. err = m.pool.WithClient(func(c *KVClient) error {
  232. return c.Write(key, string(data))
  233. })
  234. if err != nil {
  235. return fmt.Errorf("failed to write schema: %w", err)
  236. }
  237. // Update catalog
  238. if err := m.addToCatalog(schema.Name); err != nil {
  239. // Rollback schema write
  240. m.pool.WithClient(func(c *KVClient) error {
  241. return c.Delete(key)
  242. })
  243. return err
  244. }
  245. // Update cache
  246. m.cache[strings.ToLower(schema.Name)] = schema
  247. m.bumpVersionLocked()
  248. return nil
  249. }
  250. // DropTable drops a table.
  251. func (m *SchemaManager) DropTable(name string) error {
  252. tableLock := m.tableLock(name)
  253. tableLock.Lock()
  254. defer tableLock.Unlock()
  255. m.mu.Lock()
  256. defer m.mu.Unlock()
  257. key := m.schemaKey(name)
  258. // Check if table exists
  259. err := m.pool.WithClient(func(c *KVClient) error {
  260. _, err := c.Read(key)
  261. return err
  262. })
  263. if err != nil {
  264. return fmt.Errorf("table not found: %s", name)
  265. }
  266. // Delete all rows by scanning their actual keys and batch-deleting them, so
  267. // a table drop no longer leaks durable rows.
  268. if err := m.deleteKeysWithPrefix([]byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(name)))); err != nil {
  269. return err
  270. }
  271. // Delete schema
  272. err = m.pool.WithClient(func(c *KVClient) error {
  273. return c.Delete(key)
  274. })
  275. if err != nil {
  276. return fmt.Errorf("failed to delete schema: %w", err)
  277. }
  278. // Delete ROWID state.
  279. m.pool.WithClient(func(c *KVClient) error {
  280. return c.Delete(m.rowIDKey(name))
  281. })
  282. // Update catalog
  283. if err := m.removeFromCatalog(name); err != nil {
  284. return err
  285. }
  286. // Update cache
  287. tableLower := strings.ToLower(name)
  288. delete(m.cache, tableLower)
  289. m.rowIDMu.Lock()
  290. delete(m.rowIDAlloc, tableLower)
  291. m.rowIDMu.Unlock()
  292. m.bumpVersionLocked()
  293. return nil
  294. }
  295. // GetSchema retrieves a table schema.
  296. func (m *SchemaManager) GetSchema(name string) (*Schema, error) {
  297. m.mu.RLock()
  298. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  299. // Clone while still holding the read lock: the cached schema's
  300. // NextRowID field is mutated under the write lock, so cloning outside
  301. // the lock races with that mutation.
  302. cloned := cloneSchema(schema)
  303. m.mu.RUnlock()
  304. return cloned, nil
  305. }
  306. m.mu.RUnlock()
  307. m.mu.Lock()
  308. defer m.mu.Unlock()
  309. // Double-check after acquiring write lock
  310. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  311. return cloneSchema(schema), nil
  312. }
  313. key := m.schemaKey(name)
  314. var data string
  315. err := m.pool.WithClient(func(c *KVClient) error {
  316. var err error
  317. data, err = c.Read(key)
  318. return err
  319. })
  320. if err != nil {
  321. if err == ErrKeyNotFound {
  322. return nil, fmt.Errorf("table not found: %s", name)
  323. }
  324. return nil, err
  325. }
  326. var schema Schema
  327. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  328. return nil, fmt.Errorf("failed to parse schema: %w", err)
  329. }
  330. m.cache[strings.ToLower(name)] = &schema
  331. return cloneSchema(&schema), nil
  332. }
  333. func cloneSchema(schema *Schema) *Schema {
  334. if schema == nil {
  335. return nil
  336. }
  337. cloned := *schema
  338. cloned.Columns = append([]Column(nil), schema.Columns...)
  339. return &cloned
  340. }
  341. func cloneIndex(index *Index) *Index {
  342. if index == nil {
  343. return nil
  344. }
  345. cloned := *index
  346. cloned.Columns = append([]IndexColumn(nil), index.Columns...)
  347. return &cloned
  348. }
  349. // TableExists checks if a table exists.
  350. func (m *SchemaManager) TableExists(name string) bool {
  351. _, err := m.GetSchema(name)
  352. return err == nil
  353. }
  354. // ListTables returns all table names.
  355. func (m *SchemaManager) ListTables() ([]string, error) {
  356. var data string
  357. err := m.pool.WithClient(func(c *KVClient) error {
  358. var err error
  359. data, err = c.Read(m.catalogKey())
  360. return err
  361. })
  362. if err != nil {
  363. if err == ErrKeyNotFound {
  364. return nil, nil
  365. }
  366. return nil, err
  367. }
  368. var tables []string
  369. if err := json.Unmarshal([]byte(data), &tables); err != nil {
  370. return nil, fmt.Errorf("failed to parse catalog: %w", err)
  371. }
  372. return tables, nil
  373. }
  374. // addToCatalog adds a table to the catalog.
  375. func (m *SchemaManager) addToCatalog(name string) error {
  376. tables, err := m.ListTables()
  377. if err != nil && err != ErrKeyNotFound {
  378. return err
  379. }
  380. // Check if already exists
  381. lowerName := strings.ToLower(name)
  382. for _, t := range tables {
  383. if strings.ToLower(t) == lowerName {
  384. return nil
  385. }
  386. }
  387. tables = append(tables, name)
  388. data, err := json.Marshal(tables)
  389. if err != nil {
  390. return err
  391. }
  392. return m.pool.WithClient(func(c *KVClient) error {
  393. return c.Write(m.catalogKey(), string(data))
  394. })
  395. }
  396. // removeFromCatalog removes a table from the catalog.
  397. func (m *SchemaManager) removeFromCatalog(name string) error {
  398. tables, err := m.ListTables()
  399. if err != nil {
  400. return err
  401. }
  402. lowerName := strings.ToLower(name)
  403. newTables := make([]string, 0, len(tables))
  404. for _, t := range tables {
  405. if strings.ToLower(t) != lowerName {
  406. newTables = append(newTables, t)
  407. }
  408. }
  409. data, err := json.Marshal(newTables)
  410. if err != nil {
  411. return err
  412. }
  413. return m.pool.WithClient(func(c *KVClient) error {
  414. return c.Write(m.catalogKey(), string(data))
  415. })
  416. }
  417. // InvalidateCache clears the cache for a table.
  418. func (m *SchemaManager) InvalidateCache(name string) {
  419. m.mu.Lock()
  420. defer m.mu.Unlock()
  421. tableLower := strings.ToLower(name)
  422. delete(m.cache, tableLower)
  423. m.rowIDMu.Lock()
  424. delete(m.rowIDAlloc, tableLower)
  425. m.rowIDMu.Unlock()
  426. }
  427. // ToAnalyzerTableInfo converts a Schema to analyzer.TableInfo.
  428. func (s *Schema) ToAnalyzerTableInfo() *analyzer.TableInfo {
  429. info := &analyzer.TableInfo{
  430. Name: s.Name,
  431. }
  432. for _, col := range s.Columns {
  433. info.Columns = append(info.Columns, analyzer.ColumnInfo{
  434. Name: col.Name,
  435. Type: analyzer.TypeFromName(col.Type),
  436. Nullable: col.Nullable,
  437. PrimaryKey: col.PrimaryKey,
  438. TableName: s.Name,
  439. Generated: col.GeneratedExpr != "",
  440. })
  441. }
  442. return info
  443. }
  444. // GetColumn returns a column by name.
  445. func (s *Schema) GetColumn(name string) (*Column, bool) {
  446. lowerName := strings.ToLower(name)
  447. for i := range s.Columns {
  448. if strings.ToLower(s.Columns[i].Name) == lowerName {
  449. return &s.Columns[i], true
  450. }
  451. }
  452. return nil, false
  453. }
  454. // GetNextRowID gets and increments the next ROWID for a table. Allocation is
  455. // serialized per table via the table's own allocator so inserts on different
  456. // tables never contend, and no global SchemaManager lock is held across the
  457. // durable derivation scan.
  458. func (m *SchemaManager) GetNextRowID(table string) (int64, error) {
  459. alloc := m.rowIDAllocatorFor(table)
  460. alloc.mu.Lock()
  461. defer alloc.mu.Unlock()
  462. next, err := m.nextRowIDLocked(alloc, table)
  463. if err != nil {
  464. return 0, err
  465. }
  466. alloc.next = next + 1
  467. return next, nil
  468. }
  469. // UpdateMaxRowID updates the next ROWID if the provided value is higher.
  470. func (m *SchemaManager) UpdateMaxRowID(table string, rowid int64) error {
  471. alloc := m.rowIDAllocatorFor(table)
  472. alloc.mu.Lock()
  473. defer alloc.mu.Unlock()
  474. next, err := m.nextRowIDLocked(alloc, table)
  475. if err != nil {
  476. return err
  477. }
  478. if rowid >= next {
  479. alloc.next = rowid + 1
  480. }
  481. return nil
  482. }
  483. // nextRowIDLocked returns the current next ROWID, deriving it from durable rows
  484. // on first use (must hold the per-table allocator lock).
  485. func (m *SchemaManager) nextRowIDLocked(alloc *rowIDAllocator, table string) (int64, error) {
  486. if alloc.init {
  487. return alloc.next, nil
  488. }
  489. next, err := m.deriveNextRowID(table)
  490. if err != nil {
  491. return 0, err
  492. }
  493. if next < 1 {
  494. next = 1
  495. }
  496. alloc.next = next
  497. alloc.init = true
  498. return alloc.next, nil
  499. }
  500. // deriveNextRowID scans durable row keys to recover max(rowid)+1, streaming
  501. // each page through decodeRow instead of materializing every value.
  502. func (m *SchemaManager) deriveNextRowID(table string) (int64, error) {
  503. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  504. var maxRowID int64
  505. err := m.pool.WithClient(func(client *KVClient) (retErr error) {
  506. cursor, err := client.Scan(prefix)
  507. if err != nil {
  508. return err
  509. }
  510. defer func() {
  511. if err := cursor.Close(); retErr == nil {
  512. retErr = err
  513. }
  514. }()
  515. for {
  516. entries, done, err := cursor.Next()
  517. if err != nil {
  518. return err
  519. }
  520. for _, e := range entries {
  521. row, err := decodeRow(e.Value)
  522. if err != nil {
  523. return fmt.Errorf("failed to parse row while deriving ROWID: %w", err)
  524. }
  525. if rowid, ok := valueAsInt64(row["_rowid_"]); ok && rowid > maxRowID {
  526. maxRowID = rowid
  527. }
  528. }
  529. if done {
  530. return nil
  531. }
  532. }
  533. })
  534. if err != nil {
  535. return 0, err
  536. }
  537. return maxRowID + 1, nil
  538. }
  539. // deleteKeysWithPrefix streams the keys with the given prefix one page at a
  540. // time and atomically batch-deletes them, so a bulk operation never leaves
  541. // durable rows behind.
  542. func (m *SchemaManager) deleteKeysWithPrefix(prefix []byte) error {
  543. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  544. cursor, err := client.ScanKeys(prefix)
  545. if err != nil {
  546. return err
  547. }
  548. defer func() {
  549. if err := cursor.Close(); retErr == nil {
  550. retErr = err
  551. }
  552. }()
  553. ops := make([]BatchOp, 0, scanPageSize)
  554. batchBytes := 8
  555. flush := func() error {
  556. if len(ops) == 0 {
  557. return nil
  558. }
  559. if _, err := client.BatchWrite(ops, nil); err != nil {
  560. return err
  561. }
  562. ops = ops[:0]
  563. batchBytes = 8
  564. return nil
  565. }
  566. for {
  567. entries, done, err := cursor.Next()
  568. if err != nil {
  569. return err
  570. }
  571. for _, e := range entries {
  572. opBytes := 12 + len(e.Key)
  573. if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  574. if err := flush(); err != nil {
  575. return err
  576. }
  577. }
  578. ops = append(ops, BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)})
  579. batchBytes += opBytes
  580. }
  581. if done {
  582. return flush()
  583. }
  584. }
  585. })
  586. }
  587. func valueAsInt64(value interface{}) (int64, bool) {
  588. switch v := value.(type) {
  589. case int64:
  590. return v, true
  591. case int:
  592. return int64(v), true
  593. case float64:
  594. return int64(v), true
  595. default:
  596. return 0, false
  597. }
  598. }
  599. // getSchemaLocked retrieves schema (must hold lock).
  600. func (m *SchemaManager) getSchemaLocked(name string) (*Schema, error) {
  601. if schema, ok := m.cache[strings.ToLower(name)]; ok {
  602. return schema, nil
  603. }
  604. key := m.schemaKey(name)
  605. var data string
  606. err := m.pool.WithClient(func(c *KVClient) error {
  607. var err error
  608. data, err = c.Read(key)
  609. return err
  610. })
  611. if err != nil {
  612. if err == ErrKeyNotFound {
  613. return nil, fmt.Errorf("table not found: %s", name)
  614. }
  615. return nil, err
  616. }
  617. var schema Schema
  618. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  619. return nil, fmt.Errorf("failed to parse schema: %w", err)
  620. }
  621. m.cache[strings.ToLower(name)] = &schema
  622. return &schema, nil
  623. }
  624. // saveSchemaLocked saves schema (must hold lock).
  625. func (m *SchemaManager) saveSchemaLocked(schema *Schema) error {
  626. data, err := json.Marshal(schema)
  627. if err != nil {
  628. return fmt.Errorf("failed to serialize schema: %w", err)
  629. }
  630. key := m.schemaKey(schema.Name)
  631. err = m.pool.WithClient(func(c *KVClient) error {
  632. return c.Write(key, string(data))
  633. })
  634. if err != nil {
  635. return fmt.Errorf("failed to write schema: %w", err)
  636. }
  637. m.cache[strings.ToLower(schema.Name)] = schema
  638. return nil
  639. }
  640. // Index management methods
  641. // indexKey returns the key for an index.
  642. func (m *SchemaManager) indexKey(name string) string {
  643. return fmt.Sprintf("%s:index:%s", m.database, strings.ToLower(name))
  644. }
  645. // indexListKey returns the key for the index list.
  646. func (m *SchemaManager) indexListKey() string {
  647. return fmt.Sprintf("%s:indexes", m.database)
  648. }
  649. // CreateIndex creates a new index.
  650. func (m *SchemaManager) CreateIndex(index *Index) error {
  651. m.mu.Lock()
  652. defer m.mu.Unlock()
  653. // Check if index already exists
  654. key := m.indexKey(index.Name)
  655. err := m.pool.WithClient(func(c *KVClient) error {
  656. _, err := c.Read(key)
  657. return err
  658. })
  659. if err == nil {
  660. return fmt.Errorf("index already exists: %s", index.Name)
  661. }
  662. // Verify table exists
  663. if _, err := m.getSchemaLocked(index.Table); err != nil {
  664. return fmt.Errorf("table not found: %s", index.Table)
  665. }
  666. // Save index
  667. index.CreatedAt = time.Now()
  668. data, err := json.Marshal(index)
  669. if err != nil {
  670. return fmt.Errorf("failed to serialize index: %w", err)
  671. }
  672. err = m.pool.WithClient(func(c *KVClient) error {
  673. return c.Write(key, string(data))
  674. })
  675. if err != nil {
  676. return fmt.Errorf("failed to write index: %w", err)
  677. }
  678. // Add to index list
  679. if err := m.addToIndexList(index.Name); err != nil {
  680. return err
  681. }
  682. m.indexCache[strings.ToLower(index.Name)] = cloneIndex(index)
  683. m.bumpVersionLocked()
  684. return nil
  685. }
  686. // DropIndex drops an index.
  687. func (m *SchemaManager) DropIndex(name string) error {
  688. m.mu.Lock()
  689. defer m.mu.Unlock()
  690. key := m.indexKey(name)
  691. err := m.pool.WithClient(func(c *KVClient) error {
  692. return c.Delete(key)
  693. })
  694. if err != nil {
  695. return fmt.Errorf("failed to delete index: %w", err)
  696. }
  697. if err := m.removeFromIndexList(name); err != nil {
  698. return err
  699. }
  700. delete(m.indexCache, strings.ToLower(name))
  701. m.bumpVersionLocked()
  702. return nil
  703. }
  704. // IndexExists checks if an index exists.
  705. func (m *SchemaManager) IndexExists(name string) bool {
  706. m.mu.RLock()
  707. defer m.mu.RUnlock()
  708. return m.indexExistsUnsafe(name)
  709. }
  710. // indexExistsUnsafe reports whether an index definition is durable (m.mu must be
  711. // held).
  712. func (m *SchemaManager) indexExistsUnsafe(name string) bool {
  713. if _, ok := m.indexCache[strings.ToLower(name)]; ok {
  714. return true
  715. }
  716. key := m.indexKey(name)
  717. err := m.pool.WithClient(func(c *KVClient) error {
  718. _, err := c.Read(key)
  719. return err
  720. })
  721. return err == nil
  722. }
  723. // GetIndex retrieves an index by name.
  724. func (m *SchemaManager) GetIndex(name string) (*Index, error) {
  725. m.mu.Lock()
  726. defer m.mu.Unlock()
  727. cacheKey := strings.ToLower(name)
  728. if index, ok := m.indexCache[cacheKey]; ok {
  729. return cloneIndex(index), nil
  730. }
  731. key := m.indexKey(name)
  732. var data string
  733. err := m.pool.WithClient(func(c *KVClient) error {
  734. var err error
  735. data, err = c.Read(key)
  736. return err
  737. })
  738. if err != nil {
  739. if err == ErrKeyNotFound {
  740. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  741. }
  742. return nil, err
  743. }
  744. var index Index
  745. if err := json.Unmarshal([]byte(data), &index); err != nil {
  746. return nil, fmt.Errorf("failed to parse index: %w", err)
  747. }
  748. m.indexCache[cacheKey] = &index
  749. return cloneIndex(&index), nil
  750. }
  751. // ListIndexes returns all index names.
  752. func (m *SchemaManager) ListIndexes() ([]string, error) {
  753. m.mu.Lock()
  754. defer m.mu.Unlock()
  755. return m.indexListUnsafe()
  756. }
  757. // indexListUnsafe returns all index names, loading them from storage on first
  758. // use (m.mu must be held).
  759. func (m *SchemaManager) indexListUnsafe() ([]string, error) {
  760. if m.indexListCached {
  761. return append([]string(nil), m.indexListCache...), nil
  762. }
  763. key := m.indexListKey()
  764. var data string
  765. err := m.pool.WithClient(func(c *KVClient) error {
  766. var err error
  767. data, err = c.Read(key)
  768. return err
  769. })
  770. if err == ErrKeyNotFound {
  771. m.indexListCache = nil
  772. m.indexListCached = true
  773. return []string{}, nil
  774. }
  775. if err != nil {
  776. return nil, err
  777. }
  778. var indexes []string
  779. if err := json.Unmarshal([]byte(data), &indexes); err != nil {
  780. return nil, fmt.Errorf("failed to parse index list: %w", err)
  781. }
  782. m.indexListCache = append([]string(nil), indexes...)
  783. m.indexListCached = true
  784. return indexes, nil
  785. }
  786. // ListTableIndexes returns all indexes for a table.
  787. func (m *SchemaManager) ListTableIndexes(table string) ([]*Index, error) {
  788. indexes, err := m.ListIndexes()
  789. if err != nil {
  790. return nil, err
  791. }
  792. var result []*Index
  793. for _, name := range indexes {
  794. idx, err := m.GetIndex(name)
  795. if err != nil {
  796. // An index dropped concurrently is simply absent; any other error
  797. // (storage/IO or corrupt data) must not be silently swallowed.
  798. if errors.Is(err, ErrIndexNotFound) {
  799. continue
  800. }
  801. return nil, err
  802. }
  803. if strings.EqualFold(idx.Table, table) {
  804. result = append(result, idx)
  805. }
  806. }
  807. return result, nil
  808. }
  809. // addToIndexList adds an index name to the list.
  810. func (m *SchemaManager) addToIndexList(name string) error {
  811. key := m.indexListKey()
  812. var indexes []string
  813. var data string
  814. err := m.pool.WithClient(func(c *KVClient) error {
  815. var err error
  816. data, err = c.Read(key)
  817. return err
  818. })
  819. if err == nil {
  820. json.Unmarshal([]byte(data), &indexes)
  821. }
  822. indexes = append(indexes, name)
  823. newData, _ := json.Marshal(indexes)
  824. err = m.pool.WithClient(func(c *KVClient) error {
  825. return c.Write(key, string(newData))
  826. })
  827. if err == nil {
  828. m.indexListCache = append([]string(nil), indexes...)
  829. m.indexListCached = true
  830. }
  831. return err
  832. }
  833. // removeFromIndexList removes an index name from the list.
  834. func (m *SchemaManager) removeFromIndexList(name string) error {
  835. key := m.indexListKey()
  836. var indexes []string
  837. var data string
  838. err := m.pool.WithClient(func(c *KVClient) error {
  839. var err error
  840. data, err = c.Read(key)
  841. return err
  842. })
  843. if err != nil {
  844. return nil
  845. }
  846. json.Unmarshal([]byte(data), &indexes)
  847. var newIndexes []string
  848. for _, idx := range indexes {
  849. if !strings.EqualFold(idx, name) {
  850. newIndexes = append(newIndexes, idx)
  851. }
  852. }
  853. newData, _ := json.Marshal(newIndexes)
  854. err = m.pool.WithClient(func(c *KVClient) error {
  855. return c.Write(key, string(newData))
  856. })
  857. if err == nil {
  858. m.indexListCache = append([]string(nil), newIndexes...)
  859. m.indexListCached = true
  860. }
  861. return err
  862. }
  863. // AddColumn adds a new column to a table. When unique is non-nil the column's
  864. // UNIQUE index is registered as part of the same statement.
  865. //
  866. // A DEFAULT is materialized into every existing row before the schema is
  867. // published. NOT NULL, UNIQUE, and index validation all read persisted row
  868. // values, so a default that only existed in the catalog would be invisible to
  869. // them: a UNIQUE index over the new column could not see the duplicate values
  870. // every existing row shares, and an UPDATE to NULL would look non-null. The
  871. // rewrite is the same shape as DropColumn: the table's exclusive gate keeps
  872. // point writes and CAS commits out while rows are scanned, validated, and
  873. // rewritten, the transform is idempotent, and the schema is saved last so an
  874. // interrupted operation resumes against the old schema.
  875. //
  876. // The gate also makes the validation authoritative. A constant default is
  877. // deterministic, so its UNIQUE prediction can run as a cheap pre-scan. A
  878. // clock-reading default cannot be predicted: the validation must observe the
  879. // values the rewrite actually writes, so it is folded into the rewrite and a
  880. // duplicate aborts before the column and index are published. NOT NULL without
  881. // a default is rejected before any row is touched. The column and its index are
  882. // published together in one batch write, so a crash cannot leave the column
  883. // visible without the constraint that validates it. A nullable column without a
  884. // default needs no rewrite because a missing row field already reads as NULL.
  885. // Like DROP COLUMN, the statement holds the catalog lock for the whole scan, so
  886. // the cost is one table scan (plus one table scan for a constant-default UNIQUE
  887. // prediction) and a table-wide stall per ADD COLUMN that has a default or an
  888. // index.
  889. func (m *SchemaManager) AddColumn(table string, column Column, unique *Index) error {
  890. tableLock := m.tableLock(table)
  891. tableLock.Lock()
  892. defer tableLock.Unlock()
  893. m.mu.Lock()
  894. defer m.mu.Unlock()
  895. schema, err := m.getSchemaUnsafe(table)
  896. if err != nil {
  897. return err
  898. }
  899. if err := checkColumnAbsent(schema, column.Name); err != nil {
  900. return err
  901. }
  902. if unique != nil {
  903. if !strings.EqualFold(unique.Table, schema.Name) {
  904. return fmt.Errorf("index %s belongs to table %s, not %s", unique.Name, unique.Table, schema.Name)
  905. }
  906. if m.indexExistsUnsafe(unique.Name) {
  907. return fmt.Errorf("index already exists: %s", unique.Name)
  908. }
  909. // A clock-reading default is validated during the rewrite below, which
  910. // samples the same values it writes. Predicting them in a separate scan
  911. // could disagree across a second boundary and publish duplicates.
  912. if column.DefaultExpr == "" {
  913. if err := m.validateAddColumnUnique(table, column, unique); err != nil {
  914. return err
  915. }
  916. }
  917. }
  918. // A required column with nothing to write into rows that already exist
  919. // cannot be declared on a populated table.
  920. if !column.Nullable && !columnHasDefault(column) {
  921. hasRows, err := m.tableHasRows(table)
  922. if err != nil {
  923. return err
  924. }
  925. if hasRows {
  926. return &NotNullViolationError{Table: schema.Name, Column: column.Name}
  927. }
  928. }
  929. // An expression default is evaluated per rewritten row. When the column is
  930. // also UNIQUE, collect the values this rewrite writes and abort on a
  931. // duplicate, so validation cannot disagree with the published rows.
  932. var seen map[string]struct{}
  933. if unique != nil && column.DefaultExpr != "" {
  934. seen = make(map[string]struct{})
  935. }
  936. if columnHasDefault(column) {
  937. if err := m.applyRows(table, func(row Row) (bool, error) {
  938. value, present := rowColumnValue(row, column.Name)
  939. if !present {
  940. var err error
  941. value, _, err = m.columnDefaultValue(column)
  942. if err != nil {
  943. return false, err
  944. }
  945. row[column.Name] = value
  946. }
  947. if seen != nil {
  948. if value == nil {
  949. return !present, nil
  950. }
  951. encoded := encodeUniqueScalar(value)
  952. if _, duplicate := seen[encoded]; duplicate {
  953. return false, &UniqueViolationError{Constraint: unique.Name}
  954. }
  955. seen[encoded] = struct{}{}
  956. }
  957. return !present, nil
  958. }); err != nil {
  959. return err
  960. }
  961. }
  962. return m.publishAddedColumn(schema, column, unique)
  963. }
  964. // publishAddedColumn publishes a column and, when present, its unique index in
  965. // one atomic batch write (m.mu must be held): the schema snapshot, the index
  966. // definition, and the index list always become durable together or not at all.
  967. func (m *SchemaManager) publishAddedColumn(schema *Schema, column Column, unique *Index) error {
  968. // Publish a fresh snapshot instead of mutating readers' shared pointer.
  969. schema = cloneSchema(schema)
  970. schema.Columns = append(schema.Columns, column)
  971. schemaData, err := json.Marshal(schema)
  972. if err != nil {
  973. return fmt.Errorf("failed to serialize schema: %w", err)
  974. }
  975. ops := []BatchOp{{Op: batchPut, Key: []byte(m.schemaKey(schema.Name)), Value: schemaData}}
  976. var indexList []string
  977. if unique != nil {
  978. unique.CreatedAt = time.Now()
  979. indexData, err := json.Marshal(unique)
  980. if err != nil {
  981. return fmt.Errorf("failed to serialize index: %w", err)
  982. }
  983. indexList, err = m.indexListUnsafe()
  984. if err != nil {
  985. return err
  986. }
  987. indexList = append(indexList, unique.Name)
  988. listData, err := json.Marshal(indexList)
  989. if err != nil {
  990. return fmt.Errorf("failed to serialize index list: %w", err)
  991. }
  992. ops = append(ops,
  993. BatchOp{Op: batchPut, Key: []byte(m.indexKey(unique.Name)), Value: indexData},
  994. BatchOp{Op: batchPut, Key: []byte(m.indexListKey()), Value: listData},
  995. )
  996. }
  997. if err := m.pool.WithClient(func(c *KVClient) error {
  998. _, err := c.BatchWrite(ops, nil)
  999. return err
  1000. }); err != nil {
  1001. return fmt.Errorf("failed to publish column %s: %w", column.Name, err)
  1002. }
  1003. // Publish in-memory state only after the durable write.
  1004. m.cache[strings.ToLower(schema.Name)] = schema
  1005. if unique != nil {
  1006. m.indexCache[strings.ToLower(unique.Name)] = cloneIndex(unique)
  1007. m.indexListCache = append([]string(nil), indexList...)
  1008. m.indexListCached = true
  1009. }
  1010. m.bumpVersionLocked()
  1011. return nil
  1012. }
  1013. func checkColumnAbsent(schema *Schema, name string) error {
  1014. for _, col := range schema.Columns {
  1015. if strings.EqualFold(col.Name, name) {
  1016. return fmt.Errorf("column already exists: %s", name)
  1017. }
  1018. }
  1019. return nil
  1020. }
  1021. // DropColumn removes a column from a table.
  1022. func (m *SchemaManager) DropColumn(table, columnName string) error {
  1023. tableLock := m.tableLock(table)
  1024. tableLock.Lock()
  1025. defer tableLock.Unlock()
  1026. m.mu.Lock()
  1027. defer m.mu.Unlock()
  1028. schema, err := m.getSchemaUnsafe(table)
  1029. if err != nil {
  1030. return err
  1031. }
  1032. // Cannot drop primary key column
  1033. if strings.EqualFold(schema.PrimaryKey, columnName) {
  1034. return fmt.Errorf("cannot drop primary key column: %s", columnName)
  1035. }
  1036. // Find and remove column
  1037. newColumns := make([]Column, 0, len(schema.Columns)-1)
  1038. found := false
  1039. for _, col := range schema.Columns {
  1040. if strings.EqualFold(col.Name, columnName) {
  1041. found = true
  1042. continue
  1043. }
  1044. newColumns = append(newColumns, col)
  1045. }
  1046. if !found {
  1047. return fmt.Errorf("column not found: %s", columnName)
  1048. }
  1049. schema = cloneSchema(schema)
  1050. schema.Columns = newColumns
  1051. if err := m.rewriteRows(table, func(row Row) bool {
  1052. for key := range row {
  1053. if strings.EqualFold(key, columnName) {
  1054. delete(row, key)
  1055. return true
  1056. }
  1057. }
  1058. return false
  1059. }); err != nil {
  1060. return err
  1061. }
  1062. // Update schema
  1063. return m.updateSchemaUnsafe(schema)
  1064. }
  1065. // RenameTable renames a table.
  1066. func (m *SchemaManager) RenameTable(oldName, newName string) error {
  1067. m.mu.Lock()
  1068. defer m.mu.Unlock()
  1069. // Check if old table exists
  1070. schema, err := m.getSchemaUnsafe(oldName)
  1071. if err != nil {
  1072. return err
  1073. }
  1074. // Check if new table name already exists
  1075. _, err = m.getSchemaUnsafe(newName)
  1076. if err == nil {
  1077. return fmt.Errorf("table already exists: %s", newName)
  1078. }
  1079. schema = cloneSchema(schema)
  1080. // Update schema name
  1081. schema.Name = newName
  1082. // Move durable rows from the old table's data prefix to the new one. Without
  1083. // this, ALTER TABLE ... RENAME leaves every row under the old key and the
  1084. // renamed table appears empty.
  1085. if err := m.renameDataKeys(oldName, newName); err != nil {
  1086. return err
  1087. }
  1088. // Repoint indexes that belonged to the old table.
  1089. if err := m.repointIndexesLocked(oldName, newName); err != nil {
  1090. return err
  1091. }
  1092. // Build the replacement catalog before switching schema names. The schema,
  1093. // catalog, and stale ROWID state change in one batch, so a crash cannot
  1094. // leave neither table name addressable after all row chunks have moved.
  1095. tables, err := m.ListTables()
  1096. if err != nil {
  1097. return err
  1098. }
  1099. foundOld := false
  1100. for i, name := range tables {
  1101. if strings.EqualFold(name, oldName) {
  1102. tables[i] = newName
  1103. foundOld = true
  1104. break
  1105. }
  1106. }
  1107. if !foundOld {
  1108. return fmt.Errorf("table not found in catalog: %s", oldName)
  1109. }
  1110. catalogData, err := json.Marshal(tables)
  1111. if err != nil {
  1112. return err
  1113. }
  1114. schemaData, err := json.Marshal(schema)
  1115. if err != nil {
  1116. return err
  1117. }
  1118. oldKey := m.schemaKey(oldName)
  1119. newKey := m.schemaKey(newName)
  1120. err = m.pool.WithClient(func(c *KVClient) error {
  1121. _, err := c.BatchWrite([]BatchOp{
  1122. {Op: batchPut, Key: []byte(newKey), Value: schemaData},
  1123. {Op: batchDelete, Key: []byte(oldKey)},
  1124. {Op: batchDelete, Key: []byte(m.rowIDKey(oldName))},
  1125. {Op: batchPut, Key: []byte(m.catalogKey()), Value: catalogData},
  1126. }, nil)
  1127. return err
  1128. })
  1129. if err != nil {
  1130. return err
  1131. }
  1132. // Update cache
  1133. oldLower := strings.ToLower(oldName)
  1134. newLower := strings.ToLower(newName)
  1135. delete(m.cache, oldLower)
  1136. m.rowIDMu.Lock()
  1137. delete(m.rowIDAlloc, oldLower)
  1138. m.rowIDMu.Unlock()
  1139. // Update cache
  1140. m.cache[newLower] = schema
  1141. m.bumpVersionLocked()
  1142. return nil
  1143. }
  1144. // renameDataKeys moves every durable row of a table from the old name's data
  1145. // prefix to the new name's prefix in a single atomic batch per page.
  1146. func (m *SchemaManager) renameDataKeys(oldName, newName string) error {
  1147. oldPrefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(oldName)))
  1148. newPrefix := fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(newName))
  1149. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  1150. cursor, err := client.Scan(oldPrefix)
  1151. if err != nil {
  1152. return err
  1153. }
  1154. defer func() {
  1155. if cerr := cursor.Close(); retErr == nil {
  1156. retErr = cerr
  1157. }
  1158. }()
  1159. ops := make([]BatchOp, 0, scanPageSize*2)
  1160. batchBytes := 8
  1161. flush := func() error {
  1162. if len(ops) == 0 {
  1163. return nil
  1164. }
  1165. if _, err := client.BatchWrite(ops, nil); err != nil {
  1166. return err
  1167. }
  1168. ops = ops[:0]
  1169. batchBytes = 8
  1170. return nil
  1171. }
  1172. for {
  1173. entries, done, err := cursor.Next()
  1174. if err != nil {
  1175. return err
  1176. }
  1177. for _, e := range entries {
  1178. suffix := string(e.Key[len(oldPrefix):])
  1179. newKey := []byte(newPrefix + suffix)
  1180. opBytes := 24 + len(newKey) + len(e.Value) + len(e.Key)
  1181. if len(ops)+2 > maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  1182. if err := flush(); err != nil {
  1183. return err
  1184. }
  1185. }
  1186. ops = append(ops,
  1187. BatchOp{Op: batchPut, Key: newKey, Value: e.Value},
  1188. BatchOp{Op: batchDelete, Key: append([]byte(nil), e.Key...)},
  1189. )
  1190. batchBytes += opBytes
  1191. }
  1192. if done {
  1193. return flush()
  1194. }
  1195. }
  1196. })
  1197. }
  1198. // updateIndexesOfTableLocked applies mutate to the durable definition of every
  1199. // index on table and persists the definitions it changed. mutate reports
  1200. // whether it modified the definition; unmodified indexes are left untouched.
  1201. // The changed definitions are written in one batch, so a table's indexes never
  1202. // split between the old and new shape. The caller holds m.mu; it reads the
  1203. // durable index list directly rather than calling the self-locking ListIndexes
  1204. // helper.
  1205. func (m *SchemaManager) updateIndexesOfTableLocked(table string, mutate func(*Index) bool) error {
  1206. var names []string
  1207. if m.indexListCached {
  1208. names = append([]string(nil), m.indexListCache...)
  1209. } else {
  1210. var data string
  1211. err := m.pool.WithClient(func(c *KVClient) error {
  1212. var rerr error
  1213. data, rerr = c.Read(m.indexListKey())
  1214. return rerr
  1215. })
  1216. if err == ErrKeyNotFound {
  1217. return nil
  1218. }
  1219. if err != nil {
  1220. return err
  1221. }
  1222. if err := json.Unmarshal([]byte(data), &names); err != nil {
  1223. return err
  1224. }
  1225. }
  1226. type indexUpdate struct {
  1227. index Index
  1228. key string
  1229. value []byte
  1230. }
  1231. var updates []indexUpdate
  1232. for _, name := range names {
  1233. key := m.indexKey(name)
  1234. var data string
  1235. err := m.pool.WithClient(func(c *KVClient) error {
  1236. var rerr error
  1237. data, rerr = c.Read(key)
  1238. return rerr
  1239. })
  1240. if err != nil {
  1241. if err == ErrKeyNotFound {
  1242. continue
  1243. }
  1244. return err
  1245. }
  1246. var idx Index
  1247. if err := json.Unmarshal([]byte(data), &idx); err != nil {
  1248. return err
  1249. }
  1250. if !strings.EqualFold(idx.Table, table) {
  1251. continue
  1252. }
  1253. if !mutate(&idx) {
  1254. continue
  1255. }
  1256. updated, err := json.Marshal(&idx)
  1257. if err != nil {
  1258. return err
  1259. }
  1260. updates = append(updates, indexUpdate{index: idx, key: key, value: updated})
  1261. }
  1262. if len(updates) == 0 {
  1263. return nil
  1264. }
  1265. ops := make([]BatchOp, 0, len(updates))
  1266. for _, update := range updates {
  1267. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(update.key), Value: update.value})
  1268. }
  1269. if err := m.pool.WithClient(func(c *KVClient) error {
  1270. _, err := c.BatchWrite(ops, nil)
  1271. return err
  1272. }); err != nil {
  1273. return fmt.Errorf("update index definitions for table %s: %w", table, err)
  1274. }
  1275. for _, update := range updates {
  1276. if cached, ok := m.indexCache[strings.ToLower(update.index.Name)]; ok {
  1277. *cached = update.index
  1278. }
  1279. }
  1280. return nil
  1281. }
  1282. // repointIndexesLocked updates indexes whose table was renamed.
  1283. func (m *SchemaManager) repointIndexesLocked(oldName, newName string) error {
  1284. return m.updateIndexesOfTableLocked(oldName, func(idx *Index) bool {
  1285. idx.Table = newName
  1286. return true
  1287. })
  1288. }
  1289. // renameIndexColumnsLocked rewrites the stored definition of every index on
  1290. // table so a reference to oldName becomes newName. Index names stay unchanged:
  1291. // they are independent identifiers that SQLite does not derive from column
  1292. // names, and callers may have chosen them explicitly. A rename to an existing
  1293. // column name is rejected before this runs. Expression columns keep their SQL
  1294. // text; callers must reject a rename that such an expression depends on,
  1295. // because the text is not rewritten.
  1296. func (m *SchemaManager) renameIndexColumnsLocked(table, oldName, newName string) error {
  1297. return m.updateIndexesOfTableLocked(table, func(idx *Index) bool {
  1298. changed := false
  1299. for i := range idx.Columns {
  1300. // Expression columns persist SQL text, not a column reference; their
  1301. // text is left for the caller to drop and recreate the index.
  1302. if idx.Columns[i].Expression == "" && strings.EqualFold(idx.Columns[i].Name, oldName) {
  1303. idx.Columns[i].Name = newName
  1304. changed = true
  1305. }
  1306. }
  1307. return changed
  1308. })
  1309. }
  1310. // RenameColumn renames a column in a table. Column references in index
  1311. // definitions are rewritten; expression-index text is not, so a SQL caller must
  1312. // reject a rename that an expression depends on.
  1313. func (m *SchemaManager) RenameColumn(table, oldName, newName string) error {
  1314. tableLock := m.tableLock(table)
  1315. tableLock.Lock()
  1316. defer tableLock.Unlock()
  1317. m.mu.Lock()
  1318. defer m.mu.Unlock()
  1319. schema, err := m.getSchemaUnsafe(table)
  1320. if err != nil {
  1321. return err
  1322. }
  1323. // Check if new column name already exists
  1324. for _, col := range schema.Columns {
  1325. if strings.EqualFold(col.Name, newName) {
  1326. return fmt.Errorf("column already exists: %s", newName)
  1327. }
  1328. }
  1329. schema = cloneSchema(schema)
  1330. // Find and rename column
  1331. found := false
  1332. for i, col := range schema.Columns {
  1333. if strings.EqualFold(col.Name, oldName) {
  1334. schema.Columns[i].Name = newName
  1335. found = true
  1336. // Update primary key reference if needed
  1337. if strings.EqualFold(schema.PrimaryKey, oldName) {
  1338. schema.PrimaryKey = newName
  1339. }
  1340. break
  1341. }
  1342. }
  1343. if !found {
  1344. return fmt.Errorf("column not found: %s", oldName)
  1345. }
  1346. if err := m.rewriteRows(table, func(row Row) bool {
  1347. for key, value := range row {
  1348. if strings.EqualFold(key, oldName) {
  1349. delete(row, key)
  1350. row[newName] = value
  1351. return true
  1352. }
  1353. }
  1354. return false
  1355. }); err != nil {
  1356. return err
  1357. }
  1358. // Publish the renamed schema before the index definitions: reads of the new
  1359. // column name resolve against rewritten rows as soon as the schema lands.
  1360. // Index DDL is not transactional (as elsewhere in this engine), so a failure
  1361. // here leaves stale index metadata for the caller to drop and recreate.
  1362. if err := m.updateSchemaUnsafe(schema); err != nil {
  1363. return err
  1364. }
  1365. // Index metadata drives secondary-index values, UNIQUE enforcement,
  1366. // ON CONFLICT target resolution, and catalog introspection, so it must name
  1367. // the renamed column before any later statement uses the table.
  1368. return m.renameIndexColumnsLocked(table, oldName, newName)
  1369. }
  1370. // rewriteRows applies an idempotent name-keyed row transformation in bounded
  1371. // batches. Column DDL updates the schema only after all rows are rewritten, so
  1372. // an interrupted operation can safely resume against the old schema.
  1373. func (m *SchemaManager) rewriteRows(table string, transform func(Row) bool) error {
  1374. return m.applyRows(table, func(row Row) (bool, error) {
  1375. return transform(row), nil
  1376. })
  1377. }
  1378. // applyRows scans a table once and applies transform to every row. Rows for
  1379. // which transform reports a change are re-encoded and written in bounded
  1380. // batches; an error from transform aborts the scan. Callers hold the table's
  1381. // exclusive gate, so no writer can interleave between the scan and the schema
  1382. // update that follows it. Aborting after a batch was flushed is safe only
  1383. // because the DDL transforms are idempotent.
  1384. func (m *SchemaManager) applyRows(table string, transform func(Row) (bool, error)) error {
  1385. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  1386. return m.pool.WithClient(func(client *KVClient) (retErr error) {
  1387. cursor, err := client.Scan(prefix)
  1388. if err != nil {
  1389. return err
  1390. }
  1391. defer func() {
  1392. if cerr := cursor.Close(); retErr == nil {
  1393. retErr = cerr
  1394. }
  1395. }()
  1396. ops := make([]BatchOp, 0, scanPageSize)
  1397. batchBytes := 8
  1398. flush := func() error {
  1399. if len(ops) == 0 {
  1400. return nil
  1401. }
  1402. if _, err := client.BatchWrite(ops, nil); err != nil {
  1403. return err
  1404. }
  1405. ops = ops[:0]
  1406. batchBytes = 8
  1407. return nil
  1408. }
  1409. for {
  1410. entries, done, err := cursor.Next()
  1411. if err != nil {
  1412. return err
  1413. }
  1414. for _, entry := range entries {
  1415. row, err := decodeRow(entry.Value)
  1416. if err != nil {
  1417. return err
  1418. }
  1419. changed, err := transform(row)
  1420. if err != nil {
  1421. return err
  1422. }
  1423. if !changed {
  1424. continue
  1425. }
  1426. value, err := encodeRow(row)
  1427. if err != nil {
  1428. return err
  1429. }
  1430. opBytes := 12 + len(entry.Key) + len(value)
  1431. if len(ops) == maxOperations || batchBytes+opBytes > bulkBatchByteBudget {
  1432. if err := flush(); err != nil {
  1433. return err
  1434. }
  1435. }
  1436. ops = append(ops, BatchOp{Op: batchPut, Key: append([]byte(nil), entry.Key...), Value: value})
  1437. batchBytes += opBytes
  1438. }
  1439. if done {
  1440. return flush()
  1441. }
  1442. }
  1443. })
  1444. }
  1445. // validateAddColumnUnique rejects an ADD COLUMN ... UNIQUE whose backfilled
  1446. // values would duplicate across existing rows. It reads rows as they will look
  1447. // after the rewrite (a row that already carries the field from an interrupted
  1448. // earlier attempt keeps its value) and mirrors ValidateUniqueIndex for a single
  1449. // plain column: NULLs do not collide. It is used only for a constant or absent
  1450. // default; a clock-reading default is validated by the rewrite itself because
  1451. // its values cannot be predicted. The caller holds the table's exclusive gate,
  1452. // so no writer can insert a conflicting row between this check and the
  1453. // publication of the index.
  1454. func (m *SchemaManager) validateAddColumnUnique(table string, column Column, index *Index) error {
  1455. seen := make(map[string]struct{})
  1456. return m.applyRows(table, func(row Row) (bool, error) {
  1457. value, ok := rowColumnValue(row, column.Name)
  1458. if !ok {
  1459. var err error
  1460. value, _, err = m.columnDefaultValue(column)
  1461. if err != nil {
  1462. return false, err
  1463. }
  1464. }
  1465. if value == nil {
  1466. return false, nil
  1467. }
  1468. encoded := encodeUniqueScalar(value)
  1469. if _, duplicate := seen[encoded]; duplicate {
  1470. return false, &UniqueViolationError{Constraint: index.Name}
  1471. }
  1472. seen[encoded] = struct{}{}
  1473. return false, nil
  1474. })
  1475. }
  1476. // columnHasDefault reports whether the column contributes a value to rows that
  1477. // omit it, whether that default is a stored constant or an expression.
  1478. func columnHasDefault(column Column) bool {
  1479. return column.Default != nil || column.DefaultExpr != ""
  1480. }
  1481. // tableHasRows reports whether a table holds at least one durable row.
  1482. func (m *SchemaManager) tableHasRows(table string) (bool, error) {
  1483. prefix := []byte(fmt.Sprintf("%s:_data:%s:", m.database, strings.ToLower(table)))
  1484. found := false
  1485. err := m.pool.WithClient(func(client *KVClient) (retErr error) {
  1486. cursor, err := client.ScanWithLimit(prefix, 1)
  1487. if err != nil {
  1488. return err
  1489. }
  1490. defer func() {
  1491. if cerr := cursor.Close(); retErr == nil {
  1492. retErr = cerr
  1493. }
  1494. }()
  1495. entries, _, err := cursor.Next()
  1496. if err != nil {
  1497. return err
  1498. }
  1499. found = len(entries) > 0
  1500. return nil
  1501. })
  1502. return found, err
  1503. }
  1504. // getSchemaUnsafe gets a schema without locking (internal use).
  1505. func (m *SchemaManager) getSchemaUnsafe(table string) (*Schema, error) {
  1506. tableLower := strings.ToLower(table)
  1507. // Check cache
  1508. if schema, ok := m.cache[tableLower]; ok {
  1509. return schema, nil
  1510. }
  1511. // Read from storage
  1512. key := m.schemaKey(table)
  1513. var data string
  1514. err := m.pool.WithClient(func(c *KVClient) error {
  1515. var err error
  1516. data, err = c.Read(key)
  1517. return err
  1518. })
  1519. if err != nil {
  1520. return nil, fmt.Errorf("table not found: %s", table)
  1521. }
  1522. var schema Schema
  1523. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  1524. return nil, err
  1525. }
  1526. m.cache[tableLower] = &schema
  1527. return &schema, nil
  1528. }
  1529. // updateSchemaUnsafe updates a schema without locking (internal use).
  1530. func (m *SchemaManager) updateSchemaUnsafe(schema *Schema) error {
  1531. key := m.schemaKey(schema.Name)
  1532. data, _ := json.Marshal(schema)
  1533. err := m.pool.WithClient(func(c *KVClient) error {
  1534. return c.Write(key, string(data))
  1535. })
  1536. if err != nil {
  1537. return err
  1538. }
  1539. // Update cache
  1540. m.cache[strings.ToLower(schema.Name)] = schema
  1541. m.bumpVersionLocked()
  1542. return nil
  1543. }