schema_tx.go 52 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713
  1. package storage
  2. import (
  3. "fmt"
  4. "sort"
  5. "strings"
  6. "time"
  7. "github.com/goccy/go-json"
  8. )
  9. // rowRewrite is one pending ALTER TABLE row transformation. It reports whether
  10. // it changed the row and must be idempotent: it runs again on every read of the
  11. // table while the transaction is open and once more on each durable row when
  12. // the transaction commits.
  13. type rowRewrite func(row Row) (bool, error)
  14. // SchemaTx is the transaction-scoped catalog overlay of one Session. DDL inside
  15. // an explicit transaction records its catalog and row effects here instead of
  16. // publishing them: the owning session reads through the overlay, every other
  17. // session keeps reading the committed catalog, ROLLBACK discards it, and COMMIT
  18. // merges the overlay into the session's single compare-and-swap batch write, so
  19. // a multi-statement migration becomes durable atomically or not at all.
  20. //
  21. // The overlay starts out empty and reads through to the durable catalog while
  22. // recording the LSN each touched key was observed at. Those LSNs become the
  23. // commit's compare checks, so a concurrent DDL that committed after the
  24. // transaction derived its catalog makes the commit fail with ErrSerialization
  25. // instead of overwriting the concurrent change.
  26. //
  27. // Every method assumes the owning session lock is held: a Session executes one
  28. // statement at a time, so the overlay needs no lock of its own and can share the
  29. // session's row overlay directly.
  30. type SchemaTx struct {
  31. base *SchemaManager
  32. sess *Session
  33. // staged holds the transaction's definition of every touched table, keyed
  34. // by lowercased name. A nil entry means the name is not visible to this
  35. // transaction: it was dropped, or moved by RENAME TABLE.
  36. staged map[string]*Schema
  37. // stagedIndexes holds the transaction's definition of every touched index,
  38. // keyed by lowercased name; nil means dropped.
  39. stagedIndexes map[string]*Index
  40. // committedSchemas and committedIndexes cache the definitions read from
  41. // durable storage so the executor's repeated catalog lookups inside one
  42. // statement do not re-read them. An entry is dropped when the transaction
  43. // stages a replacement for the same name.
  44. committedSchemas map[string]*Schema
  45. committedIndexes map[string]*Index
  46. // tables and indexList are the staged catalog lists. They are loaded from
  47. // the committed catalog on first use and rewritten in place afterwards.
  48. tables []string
  49. indexList []string
  50. // catalogDirty and indexListDirty report that the corresponding list is
  51. // part of the commit.
  52. catalogDirty bool
  53. indexListDirty bool
  54. // reads records the durable LSN every staged catalog key was derived from
  55. // (0 for a key that was absent). commitChecks turns them into compare
  56. // checks.
  57. reads map[string]uint64
  58. // rewrites holds pending row transformations keyed by the lowercased table
  59. // name whose rows they apply to, in statement order.
  60. rewrites map[string][]rowRewrite
  61. // dataSources maps a staged table name to the durable table name whose data
  62. // prefix still holds its rows: RENAME TABLE moves the definition without
  63. // touching rows, so a later ALTER or DROP must look under the old prefix.
  64. // An entry with an empty source is a table created by this transaction.
  65. dataSources map[string]string
  66. // drops holds durable table names whose data prefix is deleted at commit.
  67. drops map[string]bool
  68. // renames maps a durable table name to the staged name its rows move to at
  69. // commit.
  70. renames map[string]string
  71. // rekeys records the overlay re-keyings this transaction performed, in
  72. // order, so a savepoint can undo them.
  73. rekeys []rekeyPair
  74. version uint64
  75. }
  76. type rekeyPair struct {
  77. from string
  78. to string
  79. }
  80. // schemaTxSnapshot is the transaction catalog state at a savepoint.
  81. type schemaTxSnapshot struct {
  82. version uint64
  83. staged map[string]*Schema
  84. stagedIndexes map[string]*Index
  85. tables []string
  86. indexList []string
  87. catalogDirty bool
  88. indexListDirty bool
  89. rewrites map[string][]rowRewrite
  90. dataSources map[string]string
  91. drops map[string]bool
  92. renames map[string]string
  93. rekeys int
  94. }
  95. func newSchemaTx(s *Session) *SchemaTx {
  96. return &SchemaTx{
  97. base: s.schema,
  98. sess: s,
  99. staged: make(map[string]*Schema),
  100. stagedIndexes: make(map[string]*Index),
  101. committedSchemas: make(map[string]*Schema),
  102. committedIndexes: make(map[string]*Index),
  103. reads: make(map[string]uint64),
  104. rewrites: make(map[string][]rowRewrite),
  105. dataSources: make(map[string]string),
  106. drops: make(map[string]bool),
  107. renames: make(map[string]string),
  108. version: s.schema.Version(),
  109. }
  110. }
  111. // snapshot captures the catalog overlay. The maps are copied shallowly: staged
  112. // definitions are cloned before they are mutated, so the captured pointers stay
  113. // immutable.
  114. func (tx *SchemaTx) snapshot() schemaTxSnapshot {
  115. snap := schemaTxSnapshot{
  116. version: tx.version,
  117. staged: make(map[string]*Schema, len(tx.staged)),
  118. stagedIndexes: make(map[string]*Index, len(tx.stagedIndexes)),
  119. tables: append([]string(nil), tx.tables...),
  120. indexList: append([]string(nil), tx.indexList...),
  121. catalogDirty: tx.catalogDirty,
  122. indexListDirty: tx.indexListDirty,
  123. rewrites: make(map[string][]rowRewrite, len(tx.rewrites)),
  124. dataSources: make(map[string]string, len(tx.dataSources)),
  125. drops: make(map[string]bool, len(tx.drops)),
  126. renames: make(map[string]string, len(tx.renames)),
  127. rekeys: len(tx.rekeys),
  128. }
  129. for k, v := range tx.staged {
  130. snap.staged[k] = v
  131. }
  132. for k, v := range tx.stagedIndexes {
  133. snap.stagedIndexes[k] = v
  134. }
  135. for k, v := range tx.rewrites {
  136. snap.rewrites[k] = append([]rowRewrite(nil), v...)
  137. }
  138. for k, v := range tx.dataSources {
  139. snap.dataSources[k] = v
  140. }
  141. for k, v := range tx.drops {
  142. snap.drops[k] = v
  143. }
  144. for k, v := range tx.renames {
  145. snap.renames[k] = v
  146. }
  147. return snap
  148. }
  149. // restore returns the overlay to a snapshot and undoes the overlay re-keyings
  150. // performed after it. Every collection is copied rather than aliased: the
  151. // snapshot is retained for a later ROLLBACK TO SAVEPOINT, and DDL staged after
  152. // this restore must not mutate the retained state. Staged schema and index
  153. // definitions are immutable once stored, so a shallow copy of the maps is
  154. // enough. The caller holds the session lock.
  155. func (tx *SchemaTx) restore(snap schemaTxSnapshot) {
  156. for i := len(tx.rekeys) - 1; i >= snap.rekeys; i-- {
  157. tx.sess.rekeyOverlayLocked(tx.rekeys[i].to, tx.rekeys[i].from)
  158. }
  159. tx.rekeys = tx.rekeys[:snap.rekeys]
  160. tx.version = snap.version
  161. tx.staged = copySchemaMap(snap.staged)
  162. tx.stagedIndexes = copyIndexMap(snap.stagedIndexes)
  163. tx.tables = append([]string(nil), snap.tables...)
  164. tx.indexList = append([]string(nil), snap.indexList...)
  165. tx.catalogDirty = snap.catalogDirty
  166. tx.indexListDirty = snap.indexListDirty
  167. tx.rewrites = copyRewriteMap(snap.rewrites)
  168. tx.dataSources = copyStringMap(snap.dataSources)
  169. tx.drops = copyBoolMap(snap.drops)
  170. tx.renames = copyStringMap(snap.renames)
  171. }
  172. func copySchemaMap(source map[string]*Schema) map[string]*Schema {
  173. cloned := make(map[string]*Schema, len(source))
  174. for key, value := range source {
  175. cloned[key] = value
  176. }
  177. return cloned
  178. }
  179. func copyIndexMap(source map[string]*Index) map[string]*Index {
  180. cloned := make(map[string]*Index, len(source))
  181. for key, value := range source {
  182. cloned[key] = value
  183. }
  184. return cloned
  185. }
  186. func copyRewriteMap(source map[string][]rowRewrite) map[string][]rowRewrite {
  187. cloned := make(map[string][]rowRewrite, len(source))
  188. for key, value := range source {
  189. cloned[key] = append([]rowRewrite(nil), value...)
  190. }
  191. return cloned
  192. }
  193. func copyStringMap(source map[string]string) map[string]string {
  194. cloned := make(map[string]string, len(source))
  195. for key, value := range source {
  196. cloned[key] = value
  197. }
  198. return cloned
  199. }
  200. func copyBoolMap(source map[string]bool) map[string]bool {
  201. cloned := make(map[string]bool, len(source))
  202. for key, value := range source {
  203. cloned[key] = value
  204. }
  205. return cloned
  206. }
  207. // bump advances the transaction's catalog version so the executor's analyzer
  208. // catalog resynchronizes after every staged change.
  209. func (tx *SchemaTx) bump() {
  210. tx.version++
  211. }
  212. func (tx *SchemaTx) recordRead(key string, lsn uint64) {
  213. if _, ok := tx.reads[key]; !ok {
  214. tx.reads[key] = lsn
  215. }
  216. }
  217. // refreshCommitted drops the cached committed definitions before the commit's
  218. // final validation. Another session's DDL may have committed while this
  219. // transaction was open, so the rows it stages must be validated against the
  220. // catalog that is current under the commit gates; the staged overlay itself is
  221. // kept. The catalog lists keep their recorded LSN: a staged list write must
  222. // still fail its compare check rather than overwrite the concurrent change.
  223. func (tx *SchemaTx) refreshCommitted() {
  224. tx.committedSchemas = make(map[string]*Schema)
  225. tx.committedIndexes = make(map[string]*Index)
  226. if !tx.catalogDirty {
  227. tx.tables = nil
  228. }
  229. if !tx.indexListDirty {
  230. tx.indexList = nil
  231. }
  232. }
  233. // getSchema returns the transaction's view of a table.
  234. func (tx *SchemaTx) getSchema(name string) (*Schema, error) {
  235. lower := strings.ToLower(name)
  236. if schema, ok := tx.staged[lower]; ok {
  237. if schema == nil {
  238. return nil, fmt.Errorf("table not found: %s", name)
  239. }
  240. return cloneSchema(schema), nil
  241. }
  242. return tx.loadSchema(name)
  243. }
  244. // loadSchema reads the committed definition and records the LSN it observed.
  245. func (tx *SchemaTx) loadSchema(name string) (*Schema, error) {
  246. lower := strings.ToLower(name)
  247. if schema, ok := tx.committedSchemas[lower]; ok {
  248. return cloneSchema(schema), nil
  249. }
  250. key := tx.base.schemaKey(name)
  251. data, lsn, found, err := tx.base.readDurable(key)
  252. if err != nil {
  253. return nil, err
  254. }
  255. tx.recordRead(key, lsn)
  256. if !found {
  257. return nil, fmt.Errorf("table not found: %s", name)
  258. }
  259. var schema Schema
  260. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  261. return nil, fmt.Errorf("failed to parse schema: %w", err)
  262. }
  263. schema.normalizePrimaryKey()
  264. tx.committedSchemas[lower] = &schema
  265. return cloneSchema(&schema), nil
  266. }
  267. func (tx *SchemaTx) tableExists(name string) bool {
  268. _, err := tx.getSchema(name)
  269. return err == nil
  270. }
  271. // listTables returns the staged table catalog in creation order.
  272. func (tx *SchemaTx) listTables() ([]string, error) {
  273. if tx.tables == nil {
  274. tables, lsn, err := tx.base.catalogTablesDurable()
  275. if err != nil {
  276. return nil, err
  277. }
  278. tx.recordRead(tx.base.catalogKey(), lsn)
  279. tx.tables = tables
  280. }
  281. return append([]string(nil), tx.tables...), nil
  282. }
  283. func (tx *SchemaTx) stageTableList(tables []string) {
  284. tx.tables = tables
  285. tx.catalogDirty = true
  286. }
  287. // getIndex returns the transaction's view of an index definition.
  288. func (tx *SchemaTx) getIndex(name string) (*Index, error) {
  289. lower := strings.ToLower(name)
  290. if index, ok := tx.stagedIndexes[lower]; ok {
  291. if index == nil {
  292. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  293. }
  294. return cloneIndex(index), nil
  295. }
  296. if index, ok := tx.committedIndexes[lower]; ok {
  297. return cloneIndex(index), nil
  298. }
  299. key := tx.base.indexKey(name)
  300. data, lsn, found, err := tx.base.readDurable(key)
  301. if err != nil {
  302. return nil, err
  303. }
  304. tx.recordRead(key, lsn)
  305. if !found {
  306. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  307. }
  308. var index Index
  309. if err := json.Unmarshal([]byte(data), &index); err != nil {
  310. return nil, fmt.Errorf("failed to parse index: %w", err)
  311. }
  312. tx.committedIndexes[lower] = &index
  313. return cloneIndex(&index), nil
  314. }
  315. func (tx *SchemaTx) indexExists(name string) bool {
  316. _, err := tx.getIndex(name)
  317. return err == nil
  318. }
  319. func (tx *SchemaTx) listIndexes() ([]string, error) {
  320. if tx.indexList == nil {
  321. indexes, lsn, err := tx.base.indexListDurable()
  322. if err != nil {
  323. return nil, err
  324. }
  325. tx.recordRead(tx.base.indexListKey(), lsn)
  326. tx.indexList = indexes
  327. }
  328. return append([]string(nil), tx.indexList...), nil
  329. }
  330. func (tx *SchemaTx) stageIndexList(indexes []string) {
  331. tx.indexList = indexes
  332. tx.indexListDirty = true
  333. }
  334. func (tx *SchemaTx) listTableIndexes(table string) ([]*Index, error) {
  335. names, err := tx.listIndexes()
  336. if err != nil {
  337. return nil, err
  338. }
  339. var result []*Index
  340. for _, name := range names {
  341. idx, err := tx.getIndex(name)
  342. if err != nil {
  343. if err == ErrIndexNotFound || strings.Contains(err.Error(), ErrIndexNotFound.Error()) {
  344. continue
  345. }
  346. return nil, err
  347. }
  348. if strings.EqualFold(idx.Table, table) {
  349. result = append(result, idx)
  350. }
  351. }
  352. return result, nil
  353. }
  354. // stageSchema records a replacement table definition.
  355. func (tx *SchemaTx) stageSchema(schema *Schema) {
  356. lower := strings.ToLower(schema.Name)
  357. delete(tx.committedSchemas, lower)
  358. tx.staged[lower] = cloneSchema(schema)
  359. if _, ok := tx.dataSources[lower]; !ok {
  360. tx.dataSources[lower] = lower
  361. }
  362. tx.recordRead(tx.base.schemaKey(schema.Name), 0)
  363. tx.bump()
  364. }
  365. // stageDroppedTable records that a name stopped being addressable.
  366. func (tx *SchemaTx) stageDroppedTable(lower string) {
  367. delete(tx.committedSchemas, lower)
  368. tx.staged[lower] = nil
  369. tx.recordRead(tx.base.schemaKey(lower), 0)
  370. tx.bump()
  371. }
  372. // createTable stages CREATE TABLE.
  373. func (tx *SchemaTx) createTable(schema *Schema) error {
  374. if tx.tableExists(schema.Name) {
  375. return fmt.Errorf("table already exists: %s", schema.Name)
  376. }
  377. lower := strings.ToLower(schema.Name)
  378. schema = cloneSchema(schema)
  379. schema.CreatedAt = time.Now()
  380. schema.normalizePrimaryKey()
  381. tx.staged[lower] = schema
  382. tx.dataSources[lower] = ""
  383. tables, err := tx.listTables()
  384. if err != nil {
  385. return err
  386. }
  387. for _, name := range tables {
  388. if strings.EqualFold(name, schema.Name) {
  389. return fmt.Errorf("table already exists: %s", schema.Name)
  390. }
  391. }
  392. tx.stageTableList(append(tables, schema.Name))
  393. tx.bump()
  394. return nil
  395. }
  396. // createIndex stages CREATE INDEX.
  397. func (tx *SchemaTx) createIndex(index *Index) error {
  398. if tx.indexExists(index.Name) {
  399. return fmt.Errorf("index already exists: %s", index.Name)
  400. }
  401. if !tx.tableExists(index.Table) {
  402. return fmt.Errorf("table not found: %s", index.Table)
  403. }
  404. index = cloneIndex(index)
  405. index.CreatedAt = time.Now()
  406. delete(tx.committedIndexes, strings.ToLower(index.Name))
  407. tx.stagedIndexes[strings.ToLower(index.Name)] = index
  408. names, err := tx.listIndexes()
  409. if err != nil {
  410. return err
  411. }
  412. tx.stageIndexList(append(names, index.Name))
  413. tx.recordRead(tx.base.indexKey(index.Name), 0)
  414. tx.bump()
  415. return nil
  416. }
  417. // dropIndex stages DROP INDEX.
  418. func (tx *SchemaTx) dropIndex(name string) error {
  419. index, err := tx.getIndex(name)
  420. if err != nil {
  421. return fmt.Errorf("index not found: %s", name)
  422. }
  423. lower := strings.ToLower(name)
  424. delete(tx.committedIndexes, lower)
  425. tx.stagedIndexes[lower] = nil
  426. names, err := tx.listIndexes()
  427. if err != nil {
  428. return err
  429. }
  430. filtered := make([]string, 0, len(names))
  431. for _, existing := range names {
  432. if !strings.EqualFold(existing, name) {
  433. filtered = append(filtered, existing)
  434. }
  435. }
  436. tx.stageIndexList(filtered)
  437. tx.recordRead(tx.base.indexKey(index.Name), 0)
  438. tx.bump()
  439. return nil
  440. }
  441. // dropTable stages DROP TABLE. The durable rows are removed by the commit; the
  442. // session's staged rows for the table are discarded immediately because the
  443. // table is no longer visible to this transaction.
  444. func (tx *SchemaTx) dropTable(name string) error {
  445. lower := strings.ToLower(name)
  446. if _, err := tx.getSchema(name); err != nil {
  447. return fmt.Errorf("table not found: %s", name)
  448. }
  449. source := tx.durableSource(lower)
  450. if source != "" {
  451. tx.drops[source] = true
  452. }
  453. delete(tx.rewrites, lower)
  454. delete(tx.dataSources, lower)
  455. // A name that was renamed inside this transaction has no durable record of
  456. // its own; the rename already deleted the original key at commit time.
  457. if _, renamed := tx.renames[lower]; !renamed {
  458. tx.recordRead(tx.base.schemaKey(lower), 0)
  459. }
  460. tx.stageDroppedTable(lower)
  461. for _, idx := range tx.tableIndexes(lower) {
  462. if err := tx.dropIndex(idx.Name); err != nil {
  463. return err
  464. }
  465. }
  466. tables, err := tx.listTables()
  467. if err != nil {
  468. return err
  469. }
  470. filtered := make([]string, 0, len(tables))
  471. for _, existing := range tables {
  472. if !strings.EqualFold(existing, name) {
  473. filtered = append(filtered, existing)
  474. }
  475. }
  476. tx.stageTableList(filtered)
  477. tx.sess.dropOverlayTableLocked(lower)
  478. tx.sess.markTableScanned(lower)
  479. return nil
  480. }
  481. // durableSource returns the committed table name whose data prefix holds the
  482. // rows of a staged table, or "" when the table was created by this transaction.
  483. func (tx *SchemaTx) durableSource(lower string) string {
  484. if source, ok := tx.dataSources[lower]; ok {
  485. return source
  486. }
  487. return lower
  488. }
  489. // tableIndexes returns the staged index definitions of a table.
  490. func (tx *SchemaTx) tableIndexes(lower string) []*Index {
  491. names, err := tx.listIndexes()
  492. if err != nil {
  493. return nil
  494. }
  495. var result []*Index
  496. for _, name := range names {
  497. idx, err := tx.getIndex(name)
  498. if err != nil || !strings.EqualFold(idx.Table, lower) {
  499. continue
  500. }
  501. result = append(result, idx)
  502. }
  503. return result
  504. }
  505. // addColumn stages ALTER TABLE ADD COLUMN. The new column, its CHECK
  506. // predicates, its UNIQUE index, and the default backfill of existing rows are
  507. // validated against the rows this session sees (durable rows plus its own
  508. // staged writes) before anything is staged, so a rejected DDL leaves the
  509. // transaction's catalog untouched.
  510. func (tx *SchemaTx) addColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
  511. schema, err := tx.getSchema(table)
  512. if err != nil {
  513. return err
  514. }
  515. if err := checkColumnAbsent(schema, column.Name); err != nil {
  516. return err
  517. }
  518. if unique != nil {
  519. if !strings.EqualFold(unique.Table, schema.Name) {
  520. return fmt.Errorf("index %s belongs to table %s, not %s", unique.Name, unique.Table, schema.Name)
  521. }
  522. if tx.indexExists(unique.Name) {
  523. return fmt.Errorf("index already exists: %s", unique.Name)
  524. }
  525. }
  526. now := tx.base.defaultNow()
  527. // Pin the table generation before scanning: a concurrent commit that lands
  528. // between the scan and the generation capture would otherwise be invisible,
  529. // so a durable row the scan did not validate could violate the new
  530. // constraint. Recording first turns that interleaving into 40001 at COMMIT.
  531. tx.sess.markTableScanned(strings.ToLower(table))
  532. // Validation covers the committed rows, mirroring the autocommit DDL path:
  533. // rows this transaction staged are re-validated against the staged schema at
  534. // commit (applyOverlaySchemaDefaults), because a staged row may still be
  535. // rewritten or removed before then.
  536. rows, err := tx.durableRows(table)
  537. if err != nil {
  538. return err
  539. }
  540. if !column.Nullable && !columnHasDefault(column) && len(rows) > 0 {
  541. return &NotNullViolationError{Table: schema.Name, Column: column.Name}
  542. }
  543. eval := tx.base.checkEvaluator()
  544. if len(checks) > 0 && eval == nil {
  545. return fmt.Errorf("cannot validate CHECK constraints on %s: no CHECK evaluator installed", table)
  546. }
  547. seen := make(map[string]struct{})
  548. for _, row := range rows {
  549. value, present := rowColumnValue(row, column.Name)
  550. if !present {
  551. if fallback, has, defErr := tx.base.columnDefaultValue(column, now); defErr != nil {
  552. return defErr
  553. } else if has {
  554. value = fallback
  555. present = true
  556. }
  557. }
  558. if len(checks) > 0 {
  559. probe := cloneRow(row)
  560. if present {
  561. probe[column.Name] = value
  562. }
  563. for _, check := range checks {
  564. holds, evalErr := eval(check.Expression, probe)
  565. if evalErr != nil {
  566. return fmt.Errorf("evaluate CHECK constraint %s on %s: %w", check.Label(), table, evalErr)
  567. }
  568. if !holds {
  569. return &CheckViolationError{Table: table, Constraint: check.Label()}
  570. }
  571. }
  572. }
  573. if unique != nil && value != nil {
  574. encoded := encodeUniqueScalar(value)
  575. if _, duplicate := seen[encoded]; duplicate {
  576. return &UniqueViolationError{Constraint: unique.Name}
  577. }
  578. seen[encoded] = struct{}{}
  579. }
  580. }
  581. schema = cloneSchema(schema)
  582. schema.Columns = append(schema.Columns, column)
  583. schema.Checks = append(schema.Checks, checks...)
  584. tx.stageSchema(schema)
  585. if unique != nil {
  586. tx.stagedIndexes[strings.ToLower(unique.Name)] = cloneIndex(unique)
  587. names, err := tx.listIndexes()
  588. if err != nil {
  589. return err
  590. }
  591. tx.stageIndexList(append(names, unique.Name))
  592. tx.recordRead(tx.base.indexKey(unique.Name), 0)
  593. }
  594. if columnHasDefault(column) {
  595. value, has, err := tx.base.columnDefaultValue(column, now)
  596. if err != nil {
  597. return err
  598. }
  599. if has {
  600. name := column.Name
  601. tx.addRewrite(table, func(row Row) (bool, error) {
  602. if _, ok := rowColumnValue(row, name); ok {
  603. return false, nil
  604. }
  605. row[name] = value
  606. return true, nil
  607. })
  608. }
  609. }
  610. return nil
  611. }
  612. // durableRows returns the committed rows of a table, ignoring this
  613. // transaction's staged rows and pending rewrites.
  614. func (tx *SchemaTx) durableRows(table string) ([]Row, error) {
  615. var rows []Row
  616. prefix := tx.sess.table.dataPrefix(tx.durableSource(strings.ToLower(table)))
  617. err := tx.sess.table.scanRowsAtPrefix(prefix, func(_ string, row Row, _ uint64) (bool, error) {
  618. rows = append(rows, row)
  619. return false, nil
  620. })
  621. return rows, err
  622. }
  623. // dropColumn stages ALTER TABLE DROP COLUMN.
  624. func (tx *SchemaTx) dropColumn(table, columnName string) error {
  625. schema, err := tx.getSchema(table)
  626. if err != nil {
  627. return err
  628. }
  629. if schema.IsPrimaryKeyColumn(columnName) {
  630. return fmt.Errorf("cannot drop primary key column: %s", columnName)
  631. }
  632. columns := make([]Column, 0, len(schema.Columns)-1)
  633. found := false
  634. for _, column := range schema.Columns {
  635. if strings.EqualFold(column.Name, columnName) {
  636. found = true
  637. continue
  638. }
  639. columns = append(columns, column)
  640. }
  641. if !found {
  642. return fmt.Errorf("column not found: %s", columnName)
  643. }
  644. schema = cloneSchema(schema)
  645. schema.Columns = columns
  646. tx.stageSchema(schema)
  647. tx.addRewrite(table, func(row Row) (bool, error) {
  648. for key := range row {
  649. if strings.EqualFold(key, columnName) {
  650. delete(row, key)
  651. return true, nil
  652. }
  653. }
  654. return false, nil
  655. })
  656. tx.sess.markTableScanned(strings.ToLower(table))
  657. return nil
  658. }
  659. // renameColumn stages ALTER TABLE RENAME COLUMN, including the column
  660. // references of the table's index definitions.
  661. func (tx *SchemaTx) renameColumn(table, oldName, newName string) error {
  662. schema, err := tx.getSchema(table)
  663. if err != nil {
  664. return err
  665. }
  666. for _, column := range schema.Columns {
  667. if strings.EqualFold(column.Name, newName) {
  668. return fmt.Errorf("column already exists: %s", newName)
  669. }
  670. }
  671. schema = cloneSchema(schema)
  672. found := false
  673. for i := range schema.Columns {
  674. if !strings.EqualFold(schema.Columns[i].Name, oldName) {
  675. continue
  676. }
  677. schema.Columns[i].Name = newName
  678. found = true
  679. for j, pk := range schema.PrimaryKeyColumns {
  680. if strings.EqualFold(pk, oldName) {
  681. schema.PrimaryKeyColumns[j] = newName
  682. }
  683. }
  684. if strings.EqualFold(schema.PrimaryKey, oldName) {
  685. schema.PrimaryKey = newName
  686. }
  687. break
  688. }
  689. if !found {
  690. return fmt.Errorf("column not found: %s", oldName)
  691. }
  692. // CHECK predicates persist SQL text and are not rewritten; the executor
  693. // rejects a rename that one depends on, matching the autocommit path.
  694. tx.stageSchema(schema)
  695. for _, idx := range tx.tableIndexes(strings.ToLower(table)) {
  696. changed := false
  697. updated := cloneIndex(idx)
  698. for i := range updated.Columns {
  699. if updated.Columns[i].Expression == "" && strings.EqualFold(updated.Columns[i].Name, oldName) {
  700. updated.Columns[i].Name = newName
  701. changed = true
  702. }
  703. }
  704. if !changed {
  705. continue
  706. }
  707. tx.stagedIndexes[strings.ToLower(updated.Name)] = updated
  708. tx.recordRead(tx.base.indexKey(updated.Name), 0)
  709. }
  710. tx.addRewrite(table, func(row Row) (bool, error) {
  711. for key, value := range row {
  712. if strings.EqualFold(key, oldName) {
  713. delete(row, key)
  714. row[newName] = value
  715. return true, nil
  716. }
  717. }
  718. return false, nil
  719. })
  720. tx.sess.markTableScanned(strings.ToLower(table))
  721. return nil
  722. }
  723. // renameTable stages ALTER TABLE RENAME TABLE. Rows stay under their committed
  724. // data prefix until commit; the session overlay is re-keyed immediately so the
  725. // transaction reads its own writes under the new name.
  726. func (tx *SchemaTx) renameTable(oldName, newName string) error {
  727. schema, err := tx.getSchema(oldName)
  728. if err != nil {
  729. return err
  730. }
  731. if tx.tableExists(newName) {
  732. return fmt.Errorf("table already exists: %s", newName)
  733. }
  734. oldLower := strings.ToLower(oldName)
  735. newLower := strings.ToLower(newName)
  736. source := tx.durableSource(oldLower)
  737. schema = cloneSchema(schema)
  738. schema.Name = newName
  739. delete(tx.committedSchemas, oldLower)
  740. delete(tx.committedSchemas, newLower)
  741. tx.staged[oldLower] = nil
  742. tx.staged[newLower] = schema
  743. if source == "" {
  744. tx.dataSources[newLower] = ""
  745. } else {
  746. tx.dataSources[newLower] = source
  747. if strings.EqualFold(source, newLower) {
  748. // Renaming back to the committed name cancels the pending move: the
  749. // rows already live under the destination prefix, so emitting a
  750. // put+delete for the same key would be order-dependent.
  751. delete(tx.renames, source)
  752. } else {
  753. tx.renames[source] = newName
  754. }
  755. }
  756. delete(tx.dataSources, oldLower)
  757. // A drop staged before the rename keeps its durable source; the rename
  758. // moves that source instead, so the drop no longer applies to it.
  759. delete(tx.drops, newLower)
  760. if rewrites, ok := tx.rewrites[oldLower]; ok {
  761. delete(tx.rewrites, oldLower)
  762. tx.rewrites[newLower] = rewrites
  763. }
  764. for _, idx := range tx.tableIndexes(oldLower) {
  765. updated := cloneIndex(idx)
  766. updated.Table = newName
  767. tx.stagedIndexes[strings.ToLower(updated.Name)] = updated
  768. tx.recordRead(tx.base.indexKey(updated.Name), 0)
  769. }
  770. tables, err := tx.listTables()
  771. if err != nil {
  772. return err
  773. }
  774. found := false
  775. for i, name := range tables {
  776. if strings.EqualFold(name, oldName) {
  777. tables[i] = newName
  778. found = true
  779. break
  780. }
  781. }
  782. if !found {
  783. return fmt.Errorf("table not found in catalog: %s", oldName)
  784. }
  785. tx.stageTableList(tables)
  786. tx.rekeys = append(tx.rekeys, rekeyPair{from: oldLower, to: newLower})
  787. tx.sess.rekeyOverlayLocked(oldLower, newLower)
  788. source = tx.durableSource(newLower)
  789. if source != "" {
  790. tx.sess.markTableScanned(source)
  791. }
  792. tx.bump()
  793. return nil
  794. }
  795. // addRewrite appends a row transformation for a table.
  796. func (tx *SchemaTx) addRewrite(table string, rewrite rowRewrite) {
  797. lower := strings.ToLower(table)
  798. tx.rewrites[lower] = append(tx.rewrites[lower], rewrite)
  799. }
  800. // rewriteRow applies every pending transformation of a table to a row.
  801. func (tx *SchemaTx) rewriteRow(table string, row Row) (bool, error) {
  802. rewrites, ok := tx.rewrites[strings.ToLower(table)]
  803. if !ok {
  804. return false, nil
  805. }
  806. changed := false
  807. for _, rewrite := range rewrites {
  808. applied, err := rewrite(row)
  809. if err != nil {
  810. return changed, err
  811. }
  812. changed = changed || applied
  813. }
  814. return changed, nil
  815. }
  816. // commitCatalogOps returns the durable catalog writes of the transaction and
  817. // the compare checks that keep them from overwriting a concurrent commit. The
  818. // caller holds the session lock and the affected table gates.
  819. func (tx *SchemaTx) commitCatalogOps() ([]BatchOp, []CompareCheck, error) {
  820. var ops []BatchOp
  821. keys := make([]string, 0, len(tx.staged)+len(tx.stagedIndexes)+2)
  822. for lower, schema := range tx.staged {
  823. if schema == nil {
  824. if _, renamed := tx.renames[lower]; renamed {
  825. // RENAME TABLE names the source through renames; the delete of
  826. // the old schema record is emitted below.
  827. continue
  828. }
  829. ops = append(ops,
  830. BatchOp{Op: batchDelete, Key: []byte(tx.base.schemaKey(lower))},
  831. BatchOp{Op: batchDelete, Key: []byte(tx.base.rowIDKey(lower))},
  832. )
  833. // The ROWID key is an in-memory allocator input, not catalog state a
  834. // concurrent statement can change, so its delete needs no compare.
  835. keys = append(keys, tx.base.schemaKey(lower))
  836. continue
  837. }
  838. data, err := json.Marshal(schema)
  839. if err != nil {
  840. return nil, nil, fmt.Errorf("failed to serialize schema: %w", err)
  841. }
  842. key := tx.base.schemaKey(schema.Name)
  843. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(key), Value: data})
  844. keys = append(keys, key)
  845. }
  846. // A staged table definition whose name replaced an older one deletes the
  847. // old record in the same batch. A rename whose destination is the committed
  848. // name (renamed away and back) has no move to publish.
  849. for source, newName := range tx.renames {
  850. if strings.EqualFold(source, newName) {
  851. continue
  852. }
  853. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.schemaKey(source))})
  854. keys = append(keys, tx.base.schemaKey(source))
  855. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.rowIDKey(source))})
  856. }
  857. for lower, index := range tx.stagedIndexes {
  858. if index == nil {
  859. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.indexKey(lower))})
  860. keys = append(keys, tx.base.indexKey(lower))
  861. continue
  862. }
  863. data, err := json.Marshal(index)
  864. if err != nil {
  865. return nil, nil, fmt.Errorf("failed to serialize index: %w", err)
  866. }
  867. key := tx.base.indexKey(index.Name)
  868. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(key), Value: data})
  869. keys = append(keys, key)
  870. }
  871. if tx.catalogDirty {
  872. data, err := json.Marshal(tx.tables)
  873. if err != nil {
  874. return nil, nil, fmt.Errorf("failed to serialize catalog: %w", err)
  875. }
  876. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(tx.base.catalogKey()), Value: data})
  877. keys = append(keys, tx.base.catalogKey())
  878. }
  879. if tx.indexListDirty {
  880. data, err := json.Marshal(tx.indexList)
  881. if err != nil {
  882. return nil, nil, fmt.Errorf("failed to serialize index list: %w", err)
  883. }
  884. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(tx.base.indexListKey()), Value: data})
  885. keys = append(keys, tx.base.indexListKey())
  886. }
  887. checks := make([]CompareCheck, 0, len(keys))
  888. seen := make(map[string]bool, len(keys))
  889. for _, key := range keys {
  890. if seen[key] {
  891. continue
  892. }
  893. seen[key] = true
  894. checks = append(checks, CompareCheck{Key: []byte(key), LSN: tx.reads[key]})
  895. }
  896. return ops, checks, nil
  897. }
  898. // commitDataOps returns the durable row writes of the transaction: dropped
  899. // tables lose every row, renamed tables move to their new prefix, and ALTER
  900. // TABLE transformations are applied to the committed row images. Rows this
  901. // session staged in its overlay are published from the overlay instead, so they
  902. // are skipped here. The caller holds the session lock and the exclusive table
  903. // gates of every affected table.
  904. func (tx *SchemaTx) commitDataOps(budget *ddlBatchBudget) ([]BatchOp, map[string]map[string]*overlayEntry, error) {
  905. entries := make(map[string]map[string]*overlayEntry)
  906. var ops []BatchOp
  907. if len(tx.drops) == 0 && len(tx.renames) == 0 && len(tx.rewrites) == 0 {
  908. return nil, nil, nil
  909. }
  910. affected := make(map[string]bool)
  911. for table := range tx.drops {
  912. affected[table] = true
  913. }
  914. for table := range tx.renames {
  915. affected[table] = true
  916. }
  917. for table := range tx.rewrites {
  918. if source := tx.durableSource(table); source != "" && !tx.drops[source] {
  919. affected[source] = true
  920. }
  921. }
  922. names := make([]string, 0, len(affected))
  923. for table := range affected {
  924. names = append(names, table)
  925. }
  926. sort.Strings(names)
  927. for _, source := range names {
  928. if tx.drops[source] {
  929. dropped, err := tx.scanRowsForDDL(source, budget, func(key string, row Row) (bool, Row, error) {
  930. if tx.sess.overlayHolds(strings.ToLower(source), key) {
  931. return false, nil, nil
  932. }
  933. return true, nil, nil
  934. })
  935. if err != nil {
  936. return nil, nil, err
  937. }
  938. ops = append(ops, dropped...)
  939. continue
  940. }
  941. newName, renamed := tx.renames[source]
  942. if renamed && strings.EqualFold(source, newName) {
  943. // A rename that returned to the committed name has no row move; the
  944. // table still owes its pending ALTER rewrites.
  945. renamed = false
  946. }
  947. if renamed {
  948. moved, movedEntries, err := tx.renameRowOps(source, newName, budget)
  949. if err != nil {
  950. return nil, nil, err
  951. }
  952. lower := strings.ToLower(newName)
  953. if entries[lower] == nil {
  954. entries[lower] = make(map[string]*overlayEntry)
  955. }
  956. for key, entry := range movedEntries {
  957. entries[lower][key] = entry
  958. }
  959. ops = append(ops, moved...)
  960. continue
  961. }
  962. rewrites := tx.rewrites[source]
  963. if len(rewrites) == 0 {
  964. continue
  965. }
  966. updated, err := tx.scanRowsForDDL(source, budget, func(key string, row Row) (bool, Row, error) {
  967. if tx.sess.overlayHolds(strings.ToLower(source), key) {
  968. return false, nil, nil
  969. }
  970. changed, err := tx.rewriteRow(source, row)
  971. if err != nil {
  972. return false, nil, err
  973. }
  974. return changed, row, nil
  975. })
  976. if err != nil {
  977. return nil, nil, err
  978. }
  979. lower := strings.ToLower(source)
  980. if entries[lower] == nil {
  981. entries[lower] = make(map[string]*overlayEntry)
  982. }
  983. for _, op := range updated {
  984. row, err := decodeRow(op.Value)
  985. if err != nil {
  986. return nil, nil, err
  987. }
  988. entries[lower][string(op.Key)] = &overlayEntry{row: row}
  989. ops = append(ops, op)
  990. }
  991. }
  992. return ops, entries, nil
  993. }
  994. // renameRowOps builds the row move of one RENAME TABLE: every committed row of
  995. // source is written under the destination prefix and its old key deleted. A row
  996. // the transaction already re-staged under the destination (an UPDATE, DELETE,
  997. // or re-INSERT after the rename) is owned by the session overlay, so only the
  998. // stale committed key is deleted; without that delete the rename would leave an
  999. // unreachable row under the dropped table name.
  1000. func (tx *SchemaTx) renameRowOps(source, newName string, budget *ddlBatchBudget) ([]BatchOp, map[string]*overlayEntry, error) {
  1001. entries := make(map[string]*overlayEntry)
  1002. var ops []BatchOp
  1003. oldPrefix := tx.sess.table.dataPrefix(source)
  1004. newPrefix := tx.sess.table.dataPrefix(newName)
  1005. lowerNew := strings.ToLower(newName)
  1006. err := tx.sess.table.scanRowsAtPrefix(oldPrefix, func(oldKey string, row Row, _ uint64) (bool, error) {
  1007. if _, err := tx.rewriteRow(newName, row); err != nil {
  1008. return true, err
  1009. }
  1010. suffix := strings.TrimPrefix(oldKey, oldPrefix)
  1011. newKey := newPrefix + suffix
  1012. if tx.sess.overlayHolds(lowerNew, newKey) {
  1013. deleted := BatchOp{Op: batchDelete, Key: []byte(oldKey)}
  1014. if err := budget.reserve(source, 1, batchOpBytes(deleted)); err != nil {
  1015. return true, err
  1016. }
  1017. ops = append(ops, deleted)
  1018. return false, nil
  1019. }
  1020. value, err := encodeRow(row)
  1021. if err != nil {
  1022. return true, err
  1023. }
  1024. put := BatchOp{Op: batchPut, Key: []byte(newKey), Value: value}
  1025. del := BatchOp{Op: batchDelete, Key: []byte(oldKey)}
  1026. if err := budget.reserve(source, 2, batchOpBytes(put)+batchOpBytes(del)); err != nil {
  1027. return true, err
  1028. }
  1029. entries[newKey] = &overlayEntry{row: row}
  1030. ops = append(ops, put, del)
  1031. return false, nil
  1032. })
  1033. return ops, entries, err
  1034. }
  1035. // ddlBatchBudget bounds the row operations one transactional DDL commit may add
  1036. // to its single compare-batch. The whole transaction is published atomically as
  1037. // one storage transaction, so it cannot page its writes the way autocommit DDL
  1038. // does: a rewrite that would exceed the operation or byte ceiling fails with an
  1039. // explicit limit error while the table is still being scanned, instead of
  1040. // buffering every row and failing inside CompareBatchWrite.
  1041. type ddlBatchBudget struct {
  1042. ops int
  1043. bytes int
  1044. }
  1045. // reserve counts one table's pending operations and fails as soon as the batch
  1046. // ceiling is crossed.
  1047. func (b *ddlBatchBudget) reserve(table string, ops, bytes int) error {
  1048. b.ops += ops
  1049. b.bytes += bytes
  1050. if b.ops > maxOperations || b.bytes > maxTransactionSize {
  1051. return &ddlBatchLimitError{Table: table, Operations: b.ops, Bytes: b.bytes}
  1052. }
  1053. return nil
  1054. }
  1055. // ddlBatchLimitError reports that a transactional DDL row rewrite cannot fit in
  1056. // the single storage transaction the commit publishes.
  1057. type ddlBatchLimitError struct {
  1058. Table string
  1059. Operations int
  1060. Bytes int
  1061. }
  1062. func (e *ddlBatchLimitError) Error() string {
  1063. return fmt.Sprintf(
  1064. "transactional DDL on %s cannot be published atomically: the rewrite needs more than the storage batch limit (%d operations / %d bytes, reached %d operations / %d bytes); run the statement outside a transaction so it can page its writes, or split the migration",
  1065. e.Table, maxOperations, maxTransactionSize, e.Operations, e.Bytes)
  1066. }
  1067. // batchOpBytes is the wire size of one operation (header, key, and value), used
  1068. // to bound a commit batch before it is encoded.
  1069. func batchOpBytes(op BatchOp) int {
  1070. return 12 + len(op.Key) + len(op.Value)
  1071. }
  1072. // scanRowsForDDL streams a durable table's rows through transform. When
  1073. // transform reports a change the row is re-encoded and returned as a put under
  1074. // its existing key; when it reports removal the key is returned as a delete.
  1075. func (tx *SchemaTx) scanRowsForDDL(table string, budget *ddlBatchBudget, transform func(key string, row Row) (write bool, replacement Row, err error)) ([]BatchOp, error) {
  1076. var ops []BatchOp
  1077. prefix := []byte(tx.sess.table.dataPrefix(table))
  1078. err := tx.base.pool.WithClient(func(client *KVClient) (retErr error) {
  1079. cursor, err := client.Scan(prefix)
  1080. if err != nil {
  1081. return err
  1082. }
  1083. defer func() {
  1084. if cerr := cursor.Close(); retErr == nil {
  1085. retErr = cerr
  1086. }
  1087. }()
  1088. for {
  1089. entries, done, err := cursor.Next()
  1090. if err != nil {
  1091. return err
  1092. }
  1093. for _, entry := range entries {
  1094. row, err := decodeRow(entry.Value)
  1095. if err != nil {
  1096. return err
  1097. }
  1098. write, replacement, err := transform(string(entry.Key), row)
  1099. if err != nil {
  1100. return err
  1101. }
  1102. if !write {
  1103. continue
  1104. }
  1105. if replacement == nil {
  1106. op := BatchOp{Op: batchDelete, Key: append([]byte(nil), entry.Key...)}
  1107. if err := budget.reserve(table, 1, batchOpBytes(op)); err != nil {
  1108. return err
  1109. }
  1110. ops = append(ops, op)
  1111. continue
  1112. }
  1113. value, err := encodeRow(replacement)
  1114. if err != nil {
  1115. return err
  1116. }
  1117. op := BatchOp{Op: batchPut, Key: append([]byte(nil), entry.Key...), Value: value}
  1118. if err := budget.reserve(table, 1, batchOpBytes(op)); err != nil {
  1119. return err
  1120. }
  1121. ops = append(ops, op)
  1122. }
  1123. if done {
  1124. return nil
  1125. }
  1126. }
  1127. })
  1128. return ops, err
  1129. }
  1130. // touchedTables lists the lowercased table names whose caches the commit must
  1131. // invalidate, including the source names rows moved away from.
  1132. func (tx *SchemaTx) touchedTables() []string {
  1133. set := make(map[string]bool)
  1134. for lower, schema := range tx.staged {
  1135. set[lower] = true
  1136. if schema != nil {
  1137. set[strings.ToLower(schema.Name)] = true
  1138. if source, ok := tx.dataSources[lower]; ok && source != "" {
  1139. set[source] = true
  1140. }
  1141. }
  1142. }
  1143. for table := range tx.drops {
  1144. set[table] = true
  1145. }
  1146. for table := range tx.renames {
  1147. set[table] = true
  1148. }
  1149. for table := range tx.rewrites {
  1150. set[table] = true
  1151. if source := tx.durableSource(table); source != "" {
  1152. set[source] = true
  1153. }
  1154. }
  1155. names := make([]string, 0, len(set))
  1156. for table := range set {
  1157. names = append(names, table)
  1158. }
  1159. sort.Strings(names)
  1160. return names
  1161. }
  1162. // touchedIndexes lists the lowercased index names whose caches the commit must
  1163. // reset.
  1164. func (tx *SchemaTx) touchedIndexes() []string {
  1165. names := make([]string, 0, len(tx.stagedIndexes))
  1166. for lower := range tx.stagedIndexes {
  1167. names = append(names, lower)
  1168. }
  1169. sort.Strings(names)
  1170. return names
  1171. }
  1172. // hasDataEffects reports whether the commit must rewrite or remove durable rows.
  1173. func (tx *SchemaTx) hasDataEffects() bool {
  1174. return len(tx.drops) > 0 || len(tx.renames) > 0 || len(tx.rewrites) > 0
  1175. }
  1176. // hasDDL reports whether the transaction staged a catalog change that the
  1177. // commit has to publish.
  1178. func (tx *SchemaTx) hasDDL() bool {
  1179. return len(tx.staged) > 0 || len(tx.stagedIndexes) > 0 || tx.catalogDirty || tx.indexListDirty || tx.hasDataEffects()
  1180. }
  1181. // Session catalog access
  1182. //
  1183. // A Session is the catalog view of one SQL connection. Outside an explicit
  1184. // transaction every accessor delegates to the shared SchemaManager; while a
  1185. // transaction is open the accessors resolve against its SchemaTx overlay, so
  1186. // the session reads its own DDL without publishing it.
  1187. // GetSchema returns the table definition this session observes.
  1188. func (s *Session) GetSchema(name string) (*Schema, error) {
  1189. s.mu.Lock()
  1190. defer s.mu.Unlock()
  1191. return s.getSchemaLocked(name)
  1192. }
  1193. // getSchemaLocked resolves a table definition (caller holds the session lock).
  1194. func (s *Session) getSchemaLocked(name string) (*Schema, error) {
  1195. if s.schemaTx != nil {
  1196. return s.schemaTx.getSchema(name)
  1197. }
  1198. return s.schema.GetSchema(name)
  1199. }
  1200. // TableExists reports whether the session can resolve a table.
  1201. func (s *Session) TableExists(name string) bool {
  1202. _, err := s.GetSchema(name)
  1203. return err == nil
  1204. }
  1205. // ListTables returns the table catalog this session observes.
  1206. func (s *Session) ListTables() ([]string, error) {
  1207. s.mu.Lock()
  1208. defer s.mu.Unlock()
  1209. if s.schemaTx != nil {
  1210. return s.schemaTx.listTables()
  1211. }
  1212. return s.schema.ListTables()
  1213. }
  1214. // GetIndex returns an index definition this session observes.
  1215. func (s *Session) GetIndex(name string) (*Index, error) {
  1216. s.mu.Lock()
  1217. defer s.mu.Unlock()
  1218. if s.schemaTx != nil {
  1219. return s.schemaTx.getIndex(name)
  1220. }
  1221. return s.schema.GetIndex(name)
  1222. }
  1223. // IndexExists reports whether the session can resolve an index.
  1224. func (s *Session) IndexExists(name string) bool {
  1225. _, err := s.GetIndex(name)
  1226. return err == nil
  1227. }
  1228. // ListIndexes returns the index catalog this session observes.
  1229. func (s *Session) ListIndexes() ([]string, error) {
  1230. s.mu.Lock()
  1231. defer s.mu.Unlock()
  1232. if s.schemaTx != nil {
  1233. return s.schemaTx.listIndexes()
  1234. }
  1235. return s.schema.ListIndexes()
  1236. }
  1237. // ListTableIndexes returns the index definitions of a table as this session
  1238. // observes them.
  1239. func (s *Session) ListTableIndexes(table string) ([]*Index, error) {
  1240. s.mu.Lock()
  1241. defer s.mu.Unlock()
  1242. if s.schemaTx != nil {
  1243. return s.schemaTx.listTableIndexes(table)
  1244. }
  1245. return s.schema.ListTableIndexes(table)
  1246. }
  1247. // Version identifies the catalog generation this session observes, including
  1248. // staged DDL, so the executor resynchronizes its analyzer catalog after a
  1249. // transactional DDL statement and again after the overlay is discarded.
  1250. func (s *Session) Version() uint64 {
  1251. s.mu.Lock()
  1252. defer s.mu.Unlock()
  1253. return s.versionLocked()
  1254. }
  1255. func (s *Session) versionLocked() uint64 {
  1256. if s.schemaTx != nil {
  1257. return s.schemaTx.version
  1258. }
  1259. return s.schema.Version()
  1260. }
  1261. // GetNextRowID allocates from the shared per-table ROWID counter. The counter is
  1262. // derived from durable rows and is not part of the catalog, so a transaction
  1263. // uses the same allocator as autocommit.
  1264. func (s *Session) GetNextRowID(table string) (int64, error) {
  1265. return s.schema.GetNextRowID(table)
  1266. }
  1267. // UpdateMaxRowID raises the shared per-table ROWID counter.
  1268. func (s *Session) UpdateMaxRowID(table string, rowid int64) error {
  1269. return s.schema.UpdateMaxRowID(table, rowid)
  1270. }
  1271. // CreateTable stages CREATE TABLE inside a transaction and publishes it
  1272. // immediately otherwise.
  1273. func (s *Session) CreateTable(schema *Schema) error {
  1274. s.mu.Lock()
  1275. defer s.mu.Unlock()
  1276. if s.schemaTx == nil {
  1277. return s.schema.CreateTable(schema)
  1278. }
  1279. return s.stageDDL(func(tx *SchemaTx) error { return tx.createTable(schema) })
  1280. }
  1281. // DropTable stages DROP TABLE inside a transaction and publishes it otherwise.
  1282. func (s *Session) DropTable(name string) error {
  1283. s.mu.Lock()
  1284. defer s.mu.Unlock()
  1285. if s.schemaTx == nil {
  1286. return s.schema.DropTable(name)
  1287. }
  1288. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropTable(name) })
  1289. }
  1290. // CreateIndex stages CREATE INDEX inside a transaction and publishes it
  1291. // otherwise. A UNIQUE index is validated against the rows the session observes,
  1292. // so a transaction that staged duplicate rows fails the statement, not the
  1293. // commit.
  1294. func (s *Session) CreateIndex(index *Index) error {
  1295. s.mu.Lock()
  1296. defer s.mu.Unlock()
  1297. if s.schemaTx == nil {
  1298. if index.Unique {
  1299. return s.table.CreateUniqueIndex(index)
  1300. }
  1301. return s.schema.CreateIndex(index)
  1302. }
  1303. return s.stageDDL(func(tx *SchemaTx) error {
  1304. if err := tx.createIndex(index); err != nil {
  1305. return err
  1306. }
  1307. if !index.Unique {
  1308. return nil
  1309. }
  1310. return s.validateUniqueIndexLocked(index)
  1311. })
  1312. }
  1313. // validateUniqueIndexLocked rejects a staged UNIQUE index whose indexed values
  1314. // already collide in the rows this session observes (caller holds the session
  1315. // lock).
  1316. func (s *Session) validateUniqueIndexLocked(index *Index) error {
  1317. rows, err := s.selectLocked(index.Table, nil)
  1318. if err != nil {
  1319. return err
  1320. }
  1321. seen := make(map[string]struct{}, len(rows))
  1322. for _, row := range rows {
  1323. encoded, isNull, err := s.table.encodeUniqueValue(row, index.Columns)
  1324. if err != nil {
  1325. return err
  1326. }
  1327. if isNull {
  1328. continue
  1329. }
  1330. if _, duplicate := seen[encoded]; duplicate {
  1331. return &UniqueViolationError{Constraint: index.Name}
  1332. }
  1333. seen[encoded] = struct{}{}
  1334. }
  1335. return nil
  1336. }
  1337. // DropIndex stages DROP INDEX inside a transaction and publishes it otherwise.
  1338. func (s *Session) DropIndex(name string) error {
  1339. s.mu.Lock()
  1340. defer s.mu.Unlock()
  1341. if s.schemaTx == nil {
  1342. return s.schema.DropIndex(name)
  1343. }
  1344. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropIndex(name) })
  1345. }
  1346. // AddColumn stages ALTER TABLE ADD COLUMN inside a transaction and publishes it
  1347. // otherwise.
  1348. func (s *Session) AddColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
  1349. s.mu.Lock()
  1350. defer s.mu.Unlock()
  1351. if s.schemaTx == nil {
  1352. return s.schema.AddColumn(table, column, checks, unique)
  1353. }
  1354. return s.stageDDL(func(tx *SchemaTx) error { return tx.addColumn(table, column, checks, unique) })
  1355. }
  1356. // DropColumn stages ALTER TABLE DROP COLUMN inside a transaction and publishes
  1357. // it otherwise.
  1358. func (s *Session) DropColumn(table, columnName string) error {
  1359. s.mu.Lock()
  1360. defer s.mu.Unlock()
  1361. if s.schemaTx == nil {
  1362. return s.schema.DropColumn(table, columnName)
  1363. }
  1364. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropColumn(table, columnName) })
  1365. }
  1366. // RenameTable stages ALTER TABLE RENAME TABLE inside a transaction and
  1367. // publishes it otherwise.
  1368. func (s *Session) RenameTable(oldName, newName string) error {
  1369. s.mu.Lock()
  1370. defer s.mu.Unlock()
  1371. if s.schemaTx == nil {
  1372. return s.schema.RenameTable(oldName, newName)
  1373. }
  1374. return s.stageDDL(func(tx *SchemaTx) error { return tx.renameTable(oldName, newName) })
  1375. }
  1376. // RenameColumn stages ALTER TABLE RENAME COLUMN inside a transaction and
  1377. // publishes it otherwise.
  1378. func (s *Session) RenameColumn(table, oldName, newName string) error {
  1379. s.mu.Lock()
  1380. defer s.mu.Unlock()
  1381. if s.schemaTx == nil {
  1382. return s.schema.RenameColumn(table, oldName, newName)
  1383. }
  1384. return s.stageDDL(func(tx *SchemaTx) error { return tx.renameColumn(table, oldName, newName) })
  1385. }
  1386. // stageDDL runs one DDL statement against the transaction overlay and restores
  1387. // the catalog, overlay, and read set if it fails, so a rejected statement
  1388. // leaves the transaction exactly as it was (caller holds the session lock).
  1389. func (s *Session) stageDDL(apply func(tx *SchemaTx) error) error {
  1390. savepoint := s.savepointLocked()
  1391. if err := apply(s.schemaTx); err != nil {
  1392. s.rollbackToSavepointLocked(savepoint)
  1393. return err
  1394. }
  1395. return nil
  1396. }
  1397. // overlayHolds reports whether the session staged the given durable key of a
  1398. // table.
  1399. func (s *Session) overlayHolds(lowerTable, key string) bool {
  1400. _, ok := s.overlay[lowerTable][key]
  1401. return ok
  1402. }
  1403. // markTableScanned pins the table's current generation as a read so the commit
  1404. // fails with a serialization error if another session writes the table between
  1405. // the DDL statement and the commit.
  1406. func (s *Session) markTableScanned(table string) {
  1407. lower := strings.ToLower(table)
  1408. if _, ok := s.scanGens[lower]; !ok {
  1409. s.scanGens[lower] = s.table.generation(lower)
  1410. }
  1411. }
  1412. // dropOverlayTableLocked discards the staged rows of a table the transaction
  1413. // dropped, including their mutation-log entries (caller holds the session lock).
  1414. func (s *Session) dropOverlayTableLocked(lower string) {
  1415. delete(s.overlay, lower)
  1416. log := s.log[:0]
  1417. for _, mutation := range s.log {
  1418. if mutation.table != lower {
  1419. log = append(log, mutation)
  1420. }
  1421. }
  1422. s.log = log
  1423. delete(s.scanGens, lower)
  1424. for key, predicate := range s.predicateGens {
  1425. if predicate.table == lower {
  1426. delete(s.predicateGens, key)
  1427. }
  1428. }
  1429. prefix := s.table.dataPrefix(lower)
  1430. for key := range s.reads {
  1431. if strings.HasPrefix(key, prefix) {
  1432. delete(s.reads, key)
  1433. }
  1434. }
  1435. }
  1436. // rekeyOverlayLocked moves the staged rows of a renamed table to the new data
  1437. // prefix so the transaction reads its own rows back under the new name
  1438. // (caller holds the session lock).
  1439. func (s *Session) rekeyOverlayLocked(oldTable, newTable string) {
  1440. oldLower := strings.ToLower(oldTable)
  1441. newLower := strings.ToLower(newTable)
  1442. if oldLower == newLower {
  1443. return
  1444. }
  1445. oldPrefix := s.table.dataPrefix(oldLower)
  1446. newPrefix := s.table.dataPrefix(newLower)
  1447. if entries, ok := s.overlay[oldLower]; ok {
  1448. delete(s.overlay, oldLower)
  1449. moved := make(map[string]*overlayEntry, len(entries))
  1450. for key, entry := range entries {
  1451. moved[newPrefix+strings.TrimPrefix(key, oldPrefix)] = entry
  1452. }
  1453. s.overlay[newLower] = moved
  1454. }
  1455. for i := range s.log {
  1456. if s.log[i].table != oldLower {
  1457. continue
  1458. }
  1459. s.log[i].table = newLower
  1460. s.log[i].key = newPrefix + strings.TrimPrefix(s.log[i].key, oldPrefix)
  1461. }
  1462. if gen, ok := s.scanGens[oldLower]; ok {
  1463. delete(s.scanGens, oldLower)
  1464. s.scanGens[newLower] = gen
  1465. }
  1466. reads := make(map[string]uint64, len(s.reads))
  1467. for key, lsn := range s.reads {
  1468. if strings.HasPrefix(key, oldPrefix) {
  1469. reads[newPrefix+strings.TrimPrefix(key, oldPrefix)] = lsn
  1470. continue
  1471. }
  1472. reads[key] = lsn
  1473. }
  1474. s.reads = reads
  1475. predicates := make(map[string]predicateRead, len(s.predicateGens))
  1476. for key, predicate := range s.predicateGens {
  1477. if predicate.table == oldLower {
  1478. predicate.table = newLower
  1479. }
  1480. if strings.HasPrefix(key, oldPrefix) {
  1481. key = newPrefix + strings.TrimPrefix(key, oldPrefix)
  1482. }
  1483. predicates[key] = predicate
  1484. }
  1485. s.predicateGens = predicates
  1486. }
  1487. // ForeignKeyParticipants reports whether a write to table can affect referential
  1488. // integrity as this session observes the catalog.
  1489. func (s *Session) ForeignKeyParticipants(table string) (bool, error) {
  1490. s.mu.Lock()
  1491. defer s.mu.Unlock()
  1492. if s.schemaTx == nil {
  1493. return s.schema.ForeignKeyParticipants(table)
  1494. }
  1495. catalog, err := s.foreignKeyCatalogLocked()
  1496. if err != nil {
  1497. return false, err
  1498. }
  1499. return catalog.participants[strings.ToLower(table)], nil
  1500. }
  1501. // ForeignKeyColumnDependency returns the constraints that keep a column in
  1502. // place, as this session observes the catalog.
  1503. func (s *Session) ForeignKeyColumnDependency(table, column string) ([]string, error) {
  1504. s.mu.Lock()
  1505. defer s.mu.Unlock()
  1506. if s.schemaTx == nil {
  1507. return s.schema.ForeignKeyColumnDependency(table, column)
  1508. }
  1509. catalog, err := s.foreignKeyCatalogLocked()
  1510. if err != nil {
  1511. return nil, err
  1512. }
  1513. schema, err := s.getSchemaLocked(table)
  1514. if err != nil {
  1515. return nil, err
  1516. }
  1517. seen := make(map[string]bool)
  1518. var labels []string
  1519. add := func(child string, fk ForeignKeyConstraint) {
  1520. label := foreignKeyReferenceName(child, fk)
  1521. if seen[label] {
  1522. return
  1523. }
  1524. seen[label] = true
  1525. labels = append(labels, label)
  1526. }
  1527. for _, fk := range schema.ForeignKeys {
  1528. for _, name := range fk.Columns {
  1529. if strings.EqualFold(name, column) {
  1530. add(schema.Name, fk)
  1531. break
  1532. }
  1533. }
  1534. }
  1535. for _, ref := range catalog.referencing[strings.ToLower(table)] {
  1536. for _, name := range ref.FK.RefColumns {
  1537. if strings.EqualFold(name, column) {
  1538. add(ref.Table, ref.FK)
  1539. break
  1540. }
  1541. }
  1542. }
  1543. sort.Strings(labels)
  1544. return labels, nil
  1545. }
  1546. // ForeignKeyTableDependency returns the constraints that name table as their
  1547. // referenced parent, as this session observes the catalog.
  1548. func (s *Session) ForeignKeyTableDependency(table string) ([]string, error) {
  1549. s.mu.Lock()
  1550. defer s.mu.Unlock()
  1551. if s.schemaTx == nil {
  1552. return s.schema.ForeignKeyTableDependency(table)
  1553. }
  1554. catalog, err := s.foreignKeyCatalogLocked()
  1555. if err != nil {
  1556. return nil, err
  1557. }
  1558. seen := make(map[string]bool)
  1559. var labels []string
  1560. for _, ref := range catalog.referencing[strings.ToLower(table)] {
  1561. label := foreignKeyReferenceName(ref.Table, ref.FK)
  1562. if seen[label] {
  1563. continue
  1564. }
  1565. seen[label] = true
  1566. labels = append(labels, label)
  1567. }
  1568. sort.Strings(labels)
  1569. return labels, nil
  1570. }
  1571. // ForeignKeyClosure returns the constrained component of the given tables as
  1572. // this session observes the catalog.
  1573. func (s *Session) ForeignKeyClosure(tables []string) (map[string]bool, error) {
  1574. s.mu.Lock()
  1575. defer s.mu.Unlock()
  1576. if s.schemaTx == nil {
  1577. return s.schema.ForeignKeyClosure(tables)
  1578. }
  1579. catalog, err := s.foreignKeyCatalogLocked()
  1580. if err != nil {
  1581. return nil, err
  1582. }
  1583. return catalog.closure(tables), nil
  1584. }
  1585. // foreignKeyCatalogLocked builds the foreign-key metadata of the session's
  1586. // catalog view (caller holds the session lock).
  1587. func (s *Session) foreignKeyCatalogLocked() (*foreignKeyCatalog, error) {
  1588. version := s.versionLocked()
  1589. if s.fkCatalog != nil && s.fkVersion == version {
  1590. return s.fkCatalog, nil
  1591. }
  1592. tables, err := s.schemaTx.listTables()
  1593. if err != nil {
  1594. return nil, err
  1595. }
  1596. catalog, err := buildForeignKeyCatalog(version, tables, func(name string) (*Schema, error) {
  1597. return s.getSchemaLocked(name)
  1598. })
  1599. if err != nil {
  1600. return nil, err
  1601. }
  1602. s.fkCatalog = catalog
  1603. s.fkVersion = version
  1604. return catalog, nil
  1605. }