schema_tx.go 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715
  1. package storage
  2. import (
  3. "fmt"
  4. "sort"
  5. "strings"
  6. "time"
  7. "github.com/goccy/go-json"
  8. )
  9. // rowRewrite is one pending ALTER TABLE row transformation. It reports whether
  10. // it changed the row and must be idempotent: it runs again on every read of the
  11. // table while the transaction is open and once more on each durable row when
  12. // the transaction commits.
  13. type rowRewrite func(row Row) (bool, error)
  14. // SchemaTx is the transaction-scoped catalog overlay of one Session. DDL inside
  15. // an explicit transaction records its catalog and row effects here instead of
  16. // publishing them: the owning session reads through the overlay, every other
  17. // session keeps reading the committed catalog, ROLLBACK discards it, and COMMIT
  18. // merges the overlay into the session's single compare-and-swap batch write, so
  19. // a multi-statement migration becomes durable atomically or not at all.
  20. //
  21. // The overlay starts out empty and reads through to the durable catalog while
  22. // recording the LSN each touched key was observed at. Those LSNs become the
  23. // commit's compare checks, so a concurrent DDL that committed after the
  24. // transaction derived its catalog makes the commit fail with ErrSerialization
  25. // instead of overwriting the concurrent change.
  26. //
  27. // Every method assumes the owning session lock is held: a Session executes one
  28. // statement at a time, so the overlay needs no lock of its own and can share the
  29. // session's row overlay directly.
  30. type SchemaTx struct {
  31. base *SchemaManager
  32. sess *Session
  33. // staged holds the transaction's definition of every touched table, keyed
  34. // by lowercased name. A nil entry means the name is not visible to this
  35. // transaction: it was dropped, or moved by RENAME TABLE.
  36. staged map[string]*Schema
  37. // stagedIndexes holds the transaction's definition of every touched index,
  38. // keyed by lowercased name; nil means dropped.
  39. stagedIndexes map[string]*Index
  40. // committedSchemas and committedIndexes cache the definitions read from
  41. // durable storage so the executor's repeated catalog lookups inside one
  42. // statement do not re-read them. An entry is dropped when the transaction
  43. // stages a replacement for the same name.
  44. committedSchemas map[string]*Schema
  45. committedIndexes map[string]*Index
  46. // tables and indexList are the staged catalog lists. They are loaded from
  47. // the committed catalog on first use and rewritten in place afterwards.
  48. tables []string
  49. indexList []string
  50. // catalogDirty and indexListDirty report that the corresponding list is
  51. // part of the commit.
  52. catalogDirty bool
  53. indexListDirty bool
  54. // reads records the durable LSN every staged catalog key was derived from
  55. // (0 for a key that was absent). commitChecks turns them into compare
  56. // checks.
  57. reads map[string]uint64
  58. // rewrites holds pending row transformations keyed by the lowercased table
  59. // name whose rows they apply to, in statement order.
  60. rewrites map[string][]rowRewrite
  61. // dataSources maps a staged table name to the durable table name whose data
  62. // prefix still holds its rows: RENAME TABLE moves the definition without
  63. // touching rows, so a later ALTER or DROP must look under the old prefix.
  64. // An entry with an empty source is a table created by this transaction.
  65. dataSources map[string]string
  66. // drops holds durable table names whose data prefix is deleted at commit.
  67. drops map[string]bool
  68. // renames maps a durable table name to the staged name its rows move to at
  69. // commit.
  70. renames map[string]string
  71. // rekeys records the overlay re-keyings this transaction performed, in
  72. // order, so a savepoint can undo them.
  73. rekeys []rekeyPair
  74. version uint64
  75. }
  76. type rekeyPair struct {
  77. from string
  78. to string
  79. }
  80. // schemaTxSnapshot is the transaction catalog state at a savepoint.
  81. type schemaTxSnapshot struct {
  82. version uint64
  83. staged map[string]*Schema
  84. stagedIndexes map[string]*Index
  85. tables []string
  86. indexList []string
  87. catalogDirty bool
  88. indexListDirty bool
  89. rewrites map[string][]rowRewrite
  90. dataSources map[string]string
  91. drops map[string]bool
  92. renames map[string]string
  93. rekeys int
  94. }
  95. func newSchemaTx(s *Session) *SchemaTx {
  96. return &SchemaTx{
  97. base: s.schema,
  98. sess: s,
  99. staged: make(map[string]*Schema),
  100. stagedIndexes: make(map[string]*Index),
  101. committedSchemas: make(map[string]*Schema),
  102. committedIndexes: make(map[string]*Index),
  103. reads: make(map[string]uint64),
  104. rewrites: make(map[string][]rowRewrite),
  105. dataSources: make(map[string]string),
  106. drops: make(map[string]bool),
  107. renames: make(map[string]string),
  108. version: s.schema.Version(),
  109. }
  110. }
  111. // snapshot captures the catalog overlay. The maps are copied shallowly: staged
  112. // definitions are cloned before they are mutated, so the captured pointers stay
  113. // immutable.
  114. func (tx *SchemaTx) snapshot() schemaTxSnapshot {
  115. snap := schemaTxSnapshot{
  116. version: tx.version,
  117. staged: make(map[string]*Schema, len(tx.staged)),
  118. stagedIndexes: make(map[string]*Index, len(tx.stagedIndexes)),
  119. tables: append([]string(nil), tx.tables...),
  120. indexList: append([]string(nil), tx.indexList...),
  121. catalogDirty: tx.catalogDirty,
  122. indexListDirty: tx.indexListDirty,
  123. rewrites: make(map[string][]rowRewrite, len(tx.rewrites)),
  124. dataSources: make(map[string]string, len(tx.dataSources)),
  125. drops: make(map[string]bool, len(tx.drops)),
  126. renames: make(map[string]string, len(tx.renames)),
  127. rekeys: len(tx.rekeys),
  128. }
  129. for k, v := range tx.staged {
  130. snap.staged[k] = v
  131. }
  132. for k, v := range tx.stagedIndexes {
  133. snap.stagedIndexes[k] = v
  134. }
  135. for k, v := range tx.rewrites {
  136. snap.rewrites[k] = append([]rowRewrite(nil), v...)
  137. }
  138. for k, v := range tx.dataSources {
  139. snap.dataSources[k] = v
  140. }
  141. for k, v := range tx.drops {
  142. snap.drops[k] = v
  143. }
  144. for k, v := range tx.renames {
  145. snap.renames[k] = v
  146. }
  147. return snap
  148. }
  149. // restore returns the overlay to a snapshot and undoes the overlay re-keyings
  150. // performed after it. Every collection is copied rather than aliased: the
  151. // snapshot is retained for a later ROLLBACK TO SAVEPOINT, and DDL staged after
  152. // this restore must not mutate the retained state. Staged schema and index
  153. // definitions are immutable once stored, so a shallow copy of the maps is
  154. // enough. The caller holds the session lock.
  155. func (tx *SchemaTx) restore(snap schemaTxSnapshot) {
  156. for i := len(tx.rekeys) - 1; i >= snap.rekeys; i-- {
  157. tx.sess.rekeyOverlayLocked(tx.rekeys[i].to, tx.rekeys[i].from)
  158. }
  159. tx.rekeys = tx.rekeys[:snap.rekeys]
  160. tx.version = snap.version
  161. tx.staged = copySchemaMap(snap.staged)
  162. tx.stagedIndexes = copyIndexMap(snap.stagedIndexes)
  163. tx.tables = append([]string(nil), snap.tables...)
  164. tx.indexList = append([]string(nil), snap.indexList...)
  165. tx.catalogDirty = snap.catalogDirty
  166. tx.indexListDirty = snap.indexListDirty
  167. tx.rewrites = copyRewriteMap(snap.rewrites)
  168. tx.dataSources = copyStringMap(snap.dataSources)
  169. tx.drops = copyBoolMap(snap.drops)
  170. tx.renames = copyStringMap(snap.renames)
  171. }
  172. func copySchemaMap(source map[string]*Schema) map[string]*Schema {
  173. cloned := make(map[string]*Schema, len(source))
  174. for key, value := range source {
  175. cloned[key] = value
  176. }
  177. return cloned
  178. }
  179. func copyIndexMap(source map[string]*Index) map[string]*Index {
  180. cloned := make(map[string]*Index, len(source))
  181. for key, value := range source {
  182. cloned[key] = value
  183. }
  184. return cloned
  185. }
  186. func copyRewriteMap(source map[string][]rowRewrite) map[string][]rowRewrite {
  187. cloned := make(map[string][]rowRewrite, len(source))
  188. for key, value := range source {
  189. cloned[key] = append([]rowRewrite(nil), value...)
  190. }
  191. return cloned
  192. }
  193. func copyStringMap(source map[string]string) map[string]string {
  194. cloned := make(map[string]string, len(source))
  195. for key, value := range source {
  196. cloned[key] = value
  197. }
  198. return cloned
  199. }
  200. func copyBoolMap(source map[string]bool) map[string]bool {
  201. cloned := make(map[string]bool, len(source))
  202. for key, value := range source {
  203. cloned[key] = value
  204. }
  205. return cloned
  206. }
  207. // bump advances the transaction's catalog version so the executor's analyzer
  208. // catalog resynchronizes after every staged change.
  209. func (tx *SchemaTx) bump() {
  210. tx.version++
  211. }
  212. func (tx *SchemaTx) recordRead(key string, lsn uint64) {
  213. if _, ok := tx.reads[key]; !ok {
  214. tx.reads[key] = lsn
  215. }
  216. }
  217. // refreshCommitted drops the cached committed definitions before the commit's
  218. // final validation. Another session's DDL may have committed while this
  219. // transaction was open, so the rows it stages must be validated against the
  220. // catalog that is current under the commit gates; the staged overlay itself is
  221. // kept. The catalog lists keep their recorded LSN: a staged list write must
  222. // still fail its compare check rather than overwrite the concurrent change.
  223. func (tx *SchemaTx) refreshCommitted() {
  224. tx.committedSchemas = make(map[string]*Schema)
  225. tx.committedIndexes = make(map[string]*Index)
  226. if !tx.catalogDirty {
  227. tx.tables = nil
  228. }
  229. if !tx.indexListDirty {
  230. tx.indexList = nil
  231. }
  232. }
  233. // getSchema returns the transaction's view of a table.
  234. func (tx *SchemaTx) getSchema(name string) (*Schema, error) {
  235. lower := strings.ToLower(name)
  236. if schema, ok := tx.staged[lower]; ok {
  237. if schema == nil {
  238. return nil, fmt.Errorf("table not found: %s", name)
  239. }
  240. return cloneSchema(schema), nil
  241. }
  242. return tx.loadSchema(name)
  243. }
  244. // loadSchema reads the committed definition and records the LSN it observed.
  245. func (tx *SchemaTx) loadSchema(name string) (*Schema, error) {
  246. lower := strings.ToLower(name)
  247. if schema, ok := tx.committedSchemas[lower]; ok {
  248. return cloneSchema(schema), nil
  249. }
  250. key := tx.base.schemaKey(name)
  251. data, lsn, found, err := tx.base.readDurable(key)
  252. if err != nil {
  253. return nil, err
  254. }
  255. tx.recordRead(key, lsn)
  256. if !found {
  257. return nil, fmt.Errorf("table not found: %s", name)
  258. }
  259. var schema Schema
  260. if err := json.Unmarshal([]byte(data), &schema); err != nil {
  261. return nil, fmt.Errorf("failed to parse schema: %w", err)
  262. }
  263. schema.normalizePrimaryKey()
  264. tx.committedSchemas[lower] = &schema
  265. return cloneSchema(&schema), nil
  266. }
  267. func (tx *SchemaTx) tableExists(name string) bool {
  268. _, err := tx.getSchema(name)
  269. return err == nil
  270. }
  271. // listTables returns the staged table catalog in creation order.
  272. func (tx *SchemaTx) listTables() ([]string, error) {
  273. if tx.tables == nil {
  274. tables, lsn, err := tx.base.catalogTablesDurable()
  275. if err != nil {
  276. return nil, err
  277. }
  278. tx.recordRead(tx.base.catalogKey(), lsn)
  279. tx.tables = tables
  280. }
  281. return append([]string(nil), tx.tables...), nil
  282. }
  283. func (tx *SchemaTx) stageTableList(tables []string) {
  284. tx.tables = tables
  285. tx.catalogDirty = true
  286. }
  287. // getIndex returns the transaction's view of an index definition.
  288. func (tx *SchemaTx) getIndex(name string) (*Index, error) {
  289. lower := strings.ToLower(name)
  290. if index, ok := tx.stagedIndexes[lower]; ok {
  291. if index == nil {
  292. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  293. }
  294. return cloneIndex(index), nil
  295. }
  296. if index, ok := tx.committedIndexes[lower]; ok {
  297. return cloneIndex(index), nil
  298. }
  299. key := tx.base.indexKey(name)
  300. data, lsn, found, err := tx.base.readDurable(key)
  301. if err != nil {
  302. return nil, err
  303. }
  304. tx.recordRead(key, lsn)
  305. if !found {
  306. return nil, fmt.Errorf("%w: %s", ErrIndexNotFound, name)
  307. }
  308. var index Index
  309. if err := json.Unmarshal([]byte(data), &index); err != nil {
  310. return nil, fmt.Errorf("failed to parse index: %w", err)
  311. }
  312. tx.committedIndexes[lower] = &index
  313. return cloneIndex(&index), nil
  314. }
  315. func (tx *SchemaTx) indexExists(name string) bool {
  316. _, err := tx.getIndex(name)
  317. return err == nil
  318. }
  319. func (tx *SchemaTx) listIndexes() ([]string, error) {
  320. if tx.indexList == nil {
  321. indexes, lsn, err := tx.base.indexListDurable()
  322. if err != nil {
  323. return nil, err
  324. }
  325. tx.recordRead(tx.base.indexListKey(), lsn)
  326. tx.indexList = indexes
  327. }
  328. return append([]string(nil), tx.indexList...), nil
  329. }
  330. func (tx *SchemaTx) stageIndexList(indexes []string) {
  331. tx.indexList = indexes
  332. tx.indexListDirty = true
  333. }
  334. func (tx *SchemaTx) listTableIndexes(table string) ([]*Index, error) {
  335. names, err := tx.listIndexes()
  336. if err != nil {
  337. return nil, err
  338. }
  339. var result []*Index
  340. for _, name := range names {
  341. idx, err := tx.getIndex(name)
  342. if err != nil {
  343. if err == ErrIndexNotFound || strings.Contains(err.Error(), ErrIndexNotFound.Error()) {
  344. continue
  345. }
  346. return nil, err
  347. }
  348. if strings.EqualFold(idx.Table, table) {
  349. result = append(result, idx)
  350. }
  351. }
  352. return result, nil
  353. }
  354. // stageSchema records a replacement table definition.
  355. func (tx *SchemaTx) stageSchema(schema *Schema) {
  356. lower := strings.ToLower(schema.Name)
  357. delete(tx.committedSchemas, lower)
  358. tx.staged[lower] = cloneSchema(schema)
  359. if _, ok := tx.dataSources[lower]; !ok {
  360. tx.dataSources[lower] = lower
  361. }
  362. tx.recordRead(tx.base.schemaKey(schema.Name), 0)
  363. tx.bump()
  364. }
  365. // stageDroppedTable records that a name stopped being addressable.
  366. func (tx *SchemaTx) stageDroppedTable(lower string) {
  367. delete(tx.committedSchemas, lower)
  368. tx.staged[lower] = nil
  369. tx.recordRead(tx.base.schemaKey(lower), 0)
  370. tx.bump()
  371. }
  372. // createTable stages CREATE TABLE.
  373. func (tx *SchemaTx) createTable(schema *Schema) error {
  374. if tx.tableExists(schema.Name) {
  375. return fmt.Errorf("table already exists: %s", schema.Name)
  376. }
  377. lower := strings.ToLower(schema.Name)
  378. schema = cloneSchema(schema)
  379. schema.CreatedAt = time.Now()
  380. schema.normalizePrimaryKey()
  381. tx.staged[lower] = schema
  382. tx.dataSources[lower] = ""
  383. tables, err := tx.listTables()
  384. if err != nil {
  385. return err
  386. }
  387. for _, name := range tables {
  388. if strings.EqualFold(name, schema.Name) {
  389. return fmt.Errorf("table already exists: %s", schema.Name)
  390. }
  391. }
  392. tx.stageTableList(append(tables, schema.Name))
  393. tx.bump()
  394. return nil
  395. }
  396. // createIndex stages CREATE INDEX.
  397. func (tx *SchemaTx) createIndex(index *Index) error {
  398. if tx.indexExists(index.Name) {
  399. return fmt.Errorf("index already exists: %s", index.Name)
  400. }
  401. if !tx.tableExists(index.Table) {
  402. return fmt.Errorf("table not found: %s", index.Table)
  403. }
  404. index = cloneIndex(index)
  405. index.CreatedAt = time.Now()
  406. delete(tx.committedIndexes, strings.ToLower(index.Name))
  407. tx.stagedIndexes[strings.ToLower(index.Name)] = index
  408. names, err := tx.listIndexes()
  409. if err != nil {
  410. return err
  411. }
  412. tx.stageIndexList(append(names, index.Name))
  413. tx.recordRead(tx.base.indexKey(index.Name), 0)
  414. tx.bump()
  415. return nil
  416. }
  417. // dropIndex stages DROP INDEX.
  418. func (tx *SchemaTx) dropIndex(name string) error {
  419. index, err := tx.getIndex(name)
  420. if err != nil {
  421. return fmt.Errorf("index not found: %s", name)
  422. }
  423. lower := strings.ToLower(name)
  424. delete(tx.committedIndexes, lower)
  425. tx.stagedIndexes[lower] = nil
  426. names, err := tx.listIndexes()
  427. if err != nil {
  428. return err
  429. }
  430. filtered := make([]string, 0, len(names))
  431. for _, existing := range names {
  432. if !strings.EqualFold(existing, name) {
  433. filtered = append(filtered, existing)
  434. }
  435. }
  436. tx.stageIndexList(filtered)
  437. tx.recordRead(tx.base.indexKey(index.Name), 0)
  438. tx.bump()
  439. return nil
  440. }
  441. // dropTable stages DROP TABLE. The durable rows are removed by the commit; the
  442. // session's staged rows for the table are discarded immediately because the
  443. // table is no longer visible to this transaction.
  444. func (tx *SchemaTx) dropTable(name string) error {
  445. lower := strings.ToLower(name)
  446. if _, err := tx.getSchema(name); err != nil {
  447. return fmt.Errorf("table not found: %s", name)
  448. }
  449. source := tx.durableSource(lower)
  450. if source != "" {
  451. tx.drops[source] = true
  452. }
  453. delete(tx.rewrites, lower)
  454. delete(tx.dataSources, lower)
  455. // A name that was renamed inside this transaction has no durable record of
  456. // its own; the rename already deleted the original key at commit time.
  457. if _, renamed := tx.renames[lower]; !renamed {
  458. tx.recordRead(tx.base.schemaKey(lower), 0)
  459. }
  460. tx.stageDroppedTable(lower)
  461. for _, idx := range tx.tableIndexes(lower) {
  462. if err := tx.dropIndex(idx.Name); err != nil {
  463. return err
  464. }
  465. }
  466. tables, err := tx.listTables()
  467. if err != nil {
  468. return err
  469. }
  470. filtered := make([]string, 0, len(tables))
  471. for _, existing := range tables {
  472. if !strings.EqualFold(existing, name) {
  473. filtered = append(filtered, existing)
  474. }
  475. }
  476. tx.stageTableList(filtered)
  477. tx.sess.dropOverlayTableLocked(lower)
  478. tx.sess.markTableScanned(lower)
  479. return nil
  480. }
  481. // durableSource returns the committed table name whose data prefix holds the
  482. // rows of a staged table, or "" when the table was created by this transaction.
  483. func (tx *SchemaTx) durableSource(lower string) string {
  484. if source, ok := tx.dataSources[lower]; ok {
  485. return source
  486. }
  487. return lower
  488. }
  489. // tableIndexes returns the staged index definitions of a table.
  490. func (tx *SchemaTx) tableIndexes(lower string) []*Index {
  491. names, err := tx.listIndexes()
  492. if err != nil {
  493. return nil
  494. }
  495. var result []*Index
  496. for _, name := range names {
  497. idx, err := tx.getIndex(name)
  498. if err != nil || !strings.EqualFold(idx.Table, lower) {
  499. continue
  500. }
  501. result = append(result, idx)
  502. }
  503. return result
  504. }
  505. // addColumn stages ALTER TABLE ADD COLUMN. The new column, its CHECK
  506. // predicates, its UNIQUE index, and the default backfill of existing rows are
  507. // validated against the rows this session sees (durable rows plus its own
  508. // staged writes) before anything is staged, so a rejected DDL leaves the
  509. // transaction's catalog untouched.
  510. func (tx *SchemaTx) addColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
  511. schema, err := tx.getSchema(table)
  512. if err != nil {
  513. return err
  514. }
  515. if err := checkColumnAbsent(schema, column.Name); err != nil {
  516. return err
  517. }
  518. if unique != nil {
  519. if !strings.EqualFold(unique.Table, schema.Name) {
  520. return fmt.Errorf("index %s belongs to table %s, not %s", unique.Name, unique.Table, schema.Name)
  521. }
  522. if tx.indexExists(unique.Name) {
  523. return fmt.Errorf("index already exists: %s", unique.Name)
  524. }
  525. }
  526. now := tx.base.defaultNow()
  527. // Pin the table generation before scanning: a concurrent commit that lands
  528. // between the scan and the generation capture would otherwise be invisible,
  529. // so a durable row the scan did not validate could violate the new
  530. // constraint. Recording first turns that interleaving into 40001 at COMMIT.
  531. tx.sess.markTableScanned(strings.ToLower(table))
  532. // Validation covers the committed rows, mirroring the autocommit DDL path:
  533. // rows this transaction staged are re-validated against the staged schema at
  534. // commit (applyOverlaySchemaDefaults), because a staged row may still be
  535. // rewritten or removed before then.
  536. rows, err := tx.durableRows(table)
  537. if err != nil {
  538. return err
  539. }
  540. if !column.Nullable && !columnHasDefault(column) && len(rows) > 0 {
  541. return &NotNullViolationError{Table: schema.Name, Column: column.Name}
  542. }
  543. eval := tx.base.checkEvaluator()
  544. if len(checks) > 0 && eval == nil {
  545. return fmt.Errorf("cannot validate CHECK constraints on %s: no CHECK evaluator installed", table)
  546. }
  547. seen := make(map[string]struct{})
  548. for _, row := range rows {
  549. value, present := rowColumnValue(row, column.Name)
  550. if !present {
  551. if fallback, has, defErr := tx.base.columnDefaultValue(column, now); defErr != nil {
  552. return defErr
  553. } else if has {
  554. value = fallback
  555. present = true
  556. }
  557. }
  558. if len(checks) > 0 {
  559. probe := cloneRow(row)
  560. if present {
  561. probe[column.Name] = value
  562. }
  563. for _, check := range checks {
  564. holds, evalErr := eval(check.Expression, probe)
  565. if evalErr != nil {
  566. return fmt.Errorf("evaluate CHECK constraint %s on %s: %w", check.Label(), table, evalErr)
  567. }
  568. if !holds {
  569. return &CheckViolationError{Table: table, Constraint: check.Label()}
  570. }
  571. }
  572. }
  573. if unique != nil && value != nil {
  574. encoded := encodeUniqueScalar(value)
  575. if _, duplicate := seen[encoded]; duplicate {
  576. return &UniqueViolationError{Constraint: unique.Name}
  577. }
  578. seen[encoded] = struct{}{}
  579. }
  580. }
  581. schema = cloneSchema(schema)
  582. schema.Columns = append(schema.Columns, column)
  583. schema.Checks = append(schema.Checks, checks...)
  584. tx.stageSchema(schema)
  585. if unique != nil {
  586. tx.stagedIndexes[strings.ToLower(unique.Name)] = cloneIndex(unique)
  587. names, err := tx.listIndexes()
  588. if err != nil {
  589. return err
  590. }
  591. tx.stageIndexList(append(names, unique.Name))
  592. tx.recordRead(tx.base.indexKey(unique.Name), 0)
  593. }
  594. if columnHasDefault(column) {
  595. value, has, err := tx.base.columnDefaultValue(column, now)
  596. if err != nil {
  597. return err
  598. }
  599. if has {
  600. name := column.Name
  601. tx.addRewrite(table, func(row Row) (bool, error) {
  602. if _, ok := rowColumnValue(row, name); ok {
  603. return false, nil
  604. }
  605. row[name] = value
  606. return true, nil
  607. })
  608. }
  609. }
  610. return nil
  611. }
  612. // durableRows returns the committed rows of a table, ignoring this
  613. // transaction's staged rows and pending rewrites.
  614. func (tx *SchemaTx) durableRows(table string) ([]Row, error) {
  615. var rows []Row
  616. prefix := tx.sess.table.dataPrefix(tx.durableSource(strings.ToLower(table)))
  617. err := tx.sess.table.scanRowsAtPrefix(prefix, func(_ string, row Row, _ uint64) (bool, error) {
  618. rows = append(rows, row)
  619. return false, nil
  620. })
  621. return rows, err
  622. }
  623. // dropColumn stages ALTER TABLE DROP COLUMN.
  624. func (tx *SchemaTx) dropColumn(table, columnName string) error {
  625. schema, err := tx.getSchema(table)
  626. if err != nil {
  627. return err
  628. }
  629. if schema.IsPrimaryKeyColumn(columnName) {
  630. return fmt.Errorf("cannot drop primary key column: %s", columnName)
  631. }
  632. columns := make([]Column, 0, len(schema.Columns)-1)
  633. found := false
  634. for _, column := range schema.Columns {
  635. if strings.EqualFold(column.Name, columnName) {
  636. found = true
  637. continue
  638. }
  639. columns = append(columns, column)
  640. }
  641. if !found {
  642. return fmt.Errorf("column not found: %s", columnName)
  643. }
  644. schema = cloneSchema(schema)
  645. schema.Columns = columns
  646. tx.stageSchema(schema)
  647. tx.addRewrite(table, func(row Row) (bool, error) {
  648. for key := range row {
  649. if strings.EqualFold(key, columnName) {
  650. delete(row, key)
  651. return true, nil
  652. }
  653. }
  654. return false, nil
  655. })
  656. tx.sess.markTableScanned(strings.ToLower(table))
  657. return nil
  658. }
  659. // renameColumn stages ALTER TABLE RENAME COLUMN, including the column
  660. // references of the table's index definitions.
  661. func (tx *SchemaTx) renameColumn(table, oldName, newName string) error {
  662. schema, err := tx.getSchema(table)
  663. if err != nil {
  664. return err
  665. }
  666. for _, column := range schema.Columns {
  667. if strings.EqualFold(column.Name, newName) {
  668. return fmt.Errorf("column already exists: %s", newName)
  669. }
  670. }
  671. schema = cloneSchema(schema)
  672. found := false
  673. for i := range schema.Columns {
  674. if !strings.EqualFold(schema.Columns[i].Name, oldName) {
  675. continue
  676. }
  677. schema.Columns[i].Name = newName
  678. found = true
  679. for j, pk := range schema.PrimaryKeyColumns {
  680. if strings.EqualFold(pk, oldName) {
  681. schema.PrimaryKeyColumns[j] = newName
  682. }
  683. }
  684. if strings.EqualFold(schema.PrimaryKey, oldName) {
  685. schema.PrimaryKey = newName
  686. }
  687. break
  688. }
  689. if !found {
  690. return fmt.Errorf("column not found: %s", oldName)
  691. }
  692. // CHECK predicates persist SQL text and are not rewritten; the executor
  693. // rejects a rename that one depends on, matching the autocommit path.
  694. tx.stageSchema(schema)
  695. for _, idx := range tx.tableIndexes(strings.ToLower(table)) {
  696. changed := false
  697. updated := cloneIndex(idx)
  698. for i := range updated.Columns {
  699. if updated.Columns[i].Expression == "" && strings.EqualFold(updated.Columns[i].Name, oldName) {
  700. updated.Columns[i].Name = newName
  701. changed = true
  702. }
  703. }
  704. if !changed {
  705. continue
  706. }
  707. tx.stagedIndexes[strings.ToLower(updated.Name)] = updated
  708. tx.recordRead(tx.base.indexKey(updated.Name), 0)
  709. }
  710. tx.addRewrite(table, func(row Row) (bool, error) {
  711. for key, value := range row {
  712. if strings.EqualFold(key, oldName) {
  713. delete(row, key)
  714. row[newName] = value
  715. return true, nil
  716. }
  717. }
  718. return false, nil
  719. })
  720. tx.sess.markTableScanned(strings.ToLower(table))
  721. return nil
  722. }
  723. // renameTable stages ALTER TABLE RENAME TABLE. Rows stay under their committed
  724. // data prefix until commit; the session overlay is re-keyed immediately so the
  725. // transaction reads its own writes under the new name.
  726. func (tx *SchemaTx) renameTable(oldName, newName string) error {
  727. schema, err := tx.getSchema(oldName)
  728. if err != nil {
  729. return err
  730. }
  731. if tx.tableExists(newName) {
  732. return fmt.Errorf("table already exists: %s", newName)
  733. }
  734. oldLower := strings.ToLower(oldName)
  735. newLower := strings.ToLower(newName)
  736. source := tx.durableSource(oldLower)
  737. schema = cloneSchema(schema)
  738. schema.Name = newName
  739. delete(tx.committedSchemas, oldLower)
  740. delete(tx.committedSchemas, newLower)
  741. tx.staged[oldLower] = nil
  742. tx.staged[newLower] = schema
  743. if source == "" {
  744. tx.dataSources[newLower] = ""
  745. } else {
  746. tx.dataSources[newLower] = source
  747. if strings.EqualFold(source, newLower) {
  748. // Renaming back to the committed name cancels the pending move: the
  749. // rows already live under the destination prefix, so emitting a
  750. // put+delete for the same key would be order-dependent.
  751. delete(tx.renames, source)
  752. } else {
  753. tx.renames[source] = newName
  754. }
  755. }
  756. delete(tx.dataSources, oldLower)
  757. // A drop staged before the rename keeps its durable source; the rename
  758. // moves that source instead, so the drop no longer applies to it.
  759. delete(tx.drops, newLower)
  760. if rewrites, ok := tx.rewrites[oldLower]; ok {
  761. delete(tx.rewrites, oldLower)
  762. tx.rewrites[newLower] = rewrites
  763. }
  764. for _, idx := range tx.tableIndexes(oldLower) {
  765. updated := cloneIndex(idx)
  766. updated.Table = newName
  767. tx.stagedIndexes[strings.ToLower(updated.Name)] = updated
  768. tx.recordRead(tx.base.indexKey(updated.Name), 0)
  769. }
  770. tables, err := tx.listTables()
  771. if err != nil {
  772. return err
  773. }
  774. found := false
  775. for i, name := range tables {
  776. if strings.EqualFold(name, oldName) {
  777. tables[i] = newName
  778. found = true
  779. break
  780. }
  781. }
  782. if !found {
  783. return fmt.Errorf("table not found in catalog: %s", oldName)
  784. }
  785. tx.stageTableList(tables)
  786. tx.rekeys = append(tx.rekeys, rekeyPair{from: oldLower, to: newLower})
  787. tx.sess.rekeyOverlayLocked(oldLower, newLower)
  788. source = tx.durableSource(newLower)
  789. if source != "" {
  790. tx.sess.markTableScanned(source)
  791. }
  792. tx.bump()
  793. return nil
  794. }
  795. // addRewrite appends a row transformation for a table.
  796. func (tx *SchemaTx) addRewrite(table string, rewrite rowRewrite) {
  797. lower := strings.ToLower(table)
  798. tx.rewrites[lower] = append(tx.rewrites[lower], rewrite)
  799. }
  800. // rewriteRow applies every pending transformation of a table to a row.
  801. func (tx *SchemaTx) rewriteRow(table string, row Row) (bool, error) {
  802. rewrites, ok := tx.rewrites[strings.ToLower(table)]
  803. if !ok {
  804. return false, nil
  805. }
  806. changed := false
  807. for _, rewrite := range rewrites {
  808. applied, err := rewrite(row)
  809. if err != nil {
  810. return changed, err
  811. }
  812. changed = changed || applied
  813. }
  814. return changed, nil
  815. }
  816. // commitCatalogOps returns the durable catalog writes of the transaction and
  817. // the compare checks that keep them from overwriting a concurrent commit. The
  818. // caller holds the session lock and the affected table gates.
  819. func (tx *SchemaTx) commitCatalogOps() ([]BatchOp, []CompareCheck, error) {
  820. var ops []BatchOp
  821. keys := make([]string, 0, len(tx.staged)+len(tx.stagedIndexes)+2)
  822. for lower, schema := range tx.staged {
  823. if schema == nil {
  824. if _, renamed := tx.renames[lower]; renamed {
  825. // RENAME TABLE names the source through renames; the delete of
  826. // the old schema record is emitted below.
  827. continue
  828. }
  829. ops = append(ops,
  830. BatchOp{Op: batchDelete, Key: []byte(tx.base.schemaKey(lower))},
  831. BatchOp{Op: batchDelete, Key: []byte(tx.base.rowIDKey(lower))},
  832. )
  833. // The ROWID key is an in-memory allocator input, not catalog state a
  834. // concurrent statement can change, so its delete needs no compare.
  835. keys = append(keys, tx.base.schemaKey(lower))
  836. continue
  837. }
  838. data, err := json.Marshal(schema)
  839. if err != nil {
  840. return nil, nil, fmt.Errorf("failed to serialize schema: %w", err)
  841. }
  842. key := tx.base.schemaKey(schema.Name)
  843. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(key), Value: data})
  844. keys = append(keys, key)
  845. }
  846. // A staged table definition whose name replaced an older one deletes the
  847. // old record in the same batch. A rename whose destination is the committed
  848. // name (renamed away and back) has no move to publish.
  849. for source, newName := range tx.renames {
  850. if strings.EqualFold(source, newName) {
  851. continue
  852. }
  853. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.schemaKey(source))})
  854. keys = append(keys, tx.base.schemaKey(source))
  855. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.rowIDKey(source))})
  856. }
  857. for lower, index := range tx.stagedIndexes {
  858. if index == nil {
  859. ops = append(ops, BatchOp{Op: batchDelete, Key: []byte(tx.base.indexKey(lower))})
  860. keys = append(keys, tx.base.indexKey(lower))
  861. continue
  862. }
  863. data, err := json.Marshal(index)
  864. if err != nil {
  865. return nil, nil, fmt.Errorf("failed to serialize index: %w", err)
  866. }
  867. key := tx.base.indexKey(index.Name)
  868. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(key), Value: data})
  869. keys = append(keys, key)
  870. }
  871. if tx.catalogDirty {
  872. data, err := json.Marshal(tx.tables)
  873. if err != nil {
  874. return nil, nil, fmt.Errorf("failed to serialize catalog: %w", err)
  875. }
  876. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(tx.base.catalogKey()), Value: data})
  877. keys = append(keys, tx.base.catalogKey())
  878. }
  879. if tx.indexListDirty {
  880. data, err := json.Marshal(tx.indexList)
  881. if err != nil {
  882. return nil, nil, fmt.Errorf("failed to serialize index list: %w", err)
  883. }
  884. ops = append(ops, BatchOp{Op: batchPut, Key: []byte(tx.base.indexListKey()), Value: data})
  885. keys = append(keys, tx.base.indexListKey())
  886. }
  887. checks := make([]CompareCheck, 0, len(keys))
  888. seen := make(map[string]bool, len(keys))
  889. for _, key := range keys {
  890. if seen[key] {
  891. continue
  892. }
  893. seen[key] = true
  894. checks = append(checks, CompareCheck{Key: []byte(key), LSN: tx.reads[key]})
  895. }
  896. return ops, checks, nil
  897. }
  898. // commitDataOps returns the durable row writes of the transaction: dropped
  899. // tables lose every row, renamed tables move to their new prefix, and ALTER
  900. // TABLE transformations are applied to the committed row images. Rows this
  901. // session staged in its overlay are published from the overlay instead, so they
  902. // are skipped here. The caller holds the session lock and the exclusive table
  903. // gates of every affected table.
  904. func (tx *SchemaTx) commitDataOps(budget *ddlBatchBudget) ([]BatchOp, map[string]map[string]*overlayEntry, error) {
  905. entries := make(map[string]map[string]*overlayEntry)
  906. var ops []BatchOp
  907. if len(tx.drops) == 0 && len(tx.renames) == 0 && len(tx.rewrites) == 0 {
  908. return nil, nil, nil
  909. }
  910. affected := make(map[string]bool)
  911. for table := range tx.drops {
  912. affected[table] = true
  913. }
  914. for table := range tx.renames {
  915. affected[table] = true
  916. }
  917. for table := range tx.rewrites {
  918. if source := tx.durableSource(table); source != "" && !tx.drops[source] {
  919. affected[source] = true
  920. }
  921. }
  922. names := make([]string, 0, len(affected))
  923. for table := range affected {
  924. names = append(names, table)
  925. }
  926. sort.Strings(names)
  927. for _, source := range names {
  928. if tx.drops[source] {
  929. dropped, err := tx.scanRowsForDDL(source, budget, func(key string, row Row) (bool, Row, error) {
  930. if tx.sess.overlayHolds(strings.ToLower(source), key) {
  931. return false, nil, nil
  932. }
  933. return true, nil, nil
  934. })
  935. if err != nil {
  936. return nil, nil, err
  937. }
  938. ops = append(ops, dropped...)
  939. continue
  940. }
  941. newName, renamed := tx.renames[source]
  942. if renamed && strings.EqualFold(source, newName) {
  943. // A rename that returned to the committed name has no row move; the
  944. // table still owes its pending ALTER rewrites.
  945. renamed = false
  946. }
  947. if renamed {
  948. moved, movedEntries, err := tx.renameRowOps(source, newName, budget)
  949. if err != nil {
  950. return nil, nil, err
  951. }
  952. lower := strings.ToLower(newName)
  953. if entries[lower] == nil {
  954. entries[lower] = make(map[string]*overlayEntry)
  955. }
  956. for key, entry := range movedEntries {
  957. entries[lower][key] = entry
  958. }
  959. ops = append(ops, moved...)
  960. continue
  961. }
  962. rewrites := tx.rewrites[source]
  963. if len(rewrites) == 0 {
  964. continue
  965. }
  966. updated, err := tx.scanRowsForDDL(source, budget, func(key string, row Row) (bool, Row, error) {
  967. if tx.sess.overlayHolds(strings.ToLower(source), key) {
  968. return false, nil, nil
  969. }
  970. changed, err := tx.rewriteRow(source, row)
  971. if err != nil {
  972. return false, nil, err
  973. }
  974. return changed, row, nil
  975. })
  976. if err != nil {
  977. return nil, nil, err
  978. }
  979. lower := strings.ToLower(source)
  980. if entries[lower] == nil {
  981. entries[lower] = make(map[string]*overlayEntry)
  982. }
  983. for _, op := range updated {
  984. row, err := decodeRow(op.Value)
  985. if err != nil {
  986. return nil, nil, err
  987. }
  988. entries[lower][string(op.Key)] = &overlayEntry{row: row}
  989. ops = append(ops, op)
  990. }
  991. }
  992. return ops, entries, nil
  993. }
  994. // renameRowOps builds the row move of one RENAME TABLE: every committed row of
  995. // source is written under the destination prefix and its old key deleted. A row
  996. // the transaction already re-staged under the destination (an UPDATE, DELETE,
  997. // or re-INSERT after the rename) is owned by the session overlay, so only the
  998. // stale committed key is deleted; without that delete the rename would leave an
  999. // unreachable row under the dropped table name.
  1000. func (tx *SchemaTx) renameRowOps(source, newName string, budget *ddlBatchBudget) ([]BatchOp, map[string]*overlayEntry, error) {
  1001. entries := make(map[string]*overlayEntry)
  1002. var ops []BatchOp
  1003. oldPrefix := tx.sess.table.dataPrefix(source)
  1004. newPrefix := tx.sess.table.dataPrefix(newName)
  1005. lowerNew := strings.ToLower(newName)
  1006. err := tx.sess.table.scanRowsAtPrefix(oldPrefix, func(oldKey string, row Row, _ uint64) (bool, error) {
  1007. if _, err := tx.rewriteRow(newName, row); err != nil {
  1008. return true, err
  1009. }
  1010. suffix := strings.TrimPrefix(oldKey, oldPrefix)
  1011. newKey := newPrefix + suffix
  1012. if tx.sess.overlayHolds(lowerNew, newKey) {
  1013. deleted := BatchOp{Op: batchDelete, Key: []byte(oldKey)}
  1014. if err := budget.reserve(source, 1, batchOpBytes(deleted)); err != nil {
  1015. return true, err
  1016. }
  1017. ops = append(ops, deleted)
  1018. return false, nil
  1019. }
  1020. value, err := encodeRow(row)
  1021. if err != nil {
  1022. return true, err
  1023. }
  1024. put := BatchOp{Op: batchPut, Key: []byte(newKey), Value: value}
  1025. del := BatchOp{Op: batchDelete, Key: []byte(oldKey)}
  1026. if err := budget.reserve(source, 2, batchOpBytes(put)+batchOpBytes(del)); err != nil {
  1027. return true, err
  1028. }
  1029. entries[newKey] = &overlayEntry{row: row}
  1030. ops = append(ops, put, del)
  1031. return false, nil
  1032. })
  1033. return ops, entries, err
  1034. }
  1035. // ddlBatchBudget bounds the row operations one transactional DDL commit may add
  1036. // to its single compare-batch. The whole transaction is published atomically as
  1037. // one storage transaction, so it cannot page its writes the way autocommit DDL
  1038. // does: a rewrite that would exceed the operation or byte ceiling fails with an
  1039. // explicit limit error while the table is still being scanned, instead of
  1040. // buffering every row and failing inside CompareBatchWrite.
  1041. type ddlBatchBudget struct {
  1042. ops int
  1043. bytes int
  1044. }
  1045. // reserve counts one table's pending operations and fails as soon as the batch
  1046. // ceiling is crossed.
  1047. func (b *ddlBatchBudget) reserve(table string, ops, bytes int) error {
  1048. b.ops += ops
  1049. b.bytes += bytes
  1050. if b.ops > maxOperations || b.bytes > maxTransactionSize {
  1051. return &ddlBatchLimitError{Table: table, Operations: b.ops, Bytes: b.bytes}
  1052. }
  1053. return nil
  1054. }
  1055. // ddlBatchLimitError reports that a transactional DDL row rewrite cannot fit in
  1056. // the single storage transaction the commit publishes.
  1057. type ddlBatchLimitError struct {
  1058. Table string
  1059. Operations int
  1060. Bytes int
  1061. }
  1062. func (e *ddlBatchLimitError) Error() string {
  1063. return fmt.Sprintf(
  1064. "transactional DDL on %s cannot be published atomically: the rewrite needs more than the storage batch limit (%d operations / %d bytes, reached %d operations / %d bytes); run the statement outside a transaction so it can page its writes, or split the migration",
  1065. e.Table, maxOperations, maxTransactionSize, e.Operations, e.Bytes)
  1066. }
  1067. // batchOpBytes is the wire size of one operation (header, key, and value), used
  1068. // to bound a commit batch before it is encoded.
  1069. func batchOpBytes(op BatchOp) int {
  1070. return 12 + len(op.Key) + len(op.Value)
  1071. }
  1072. // scanRowsForDDL streams a durable table's rows through transform. When
  1073. // transform reports a change the row is re-encoded and returned as a put under
  1074. // its existing key; when it reports removal the key is returned as a delete.
  1075. func (tx *SchemaTx) scanRowsForDDL(table string, budget *ddlBatchBudget, transform func(key string, row Row) (write bool, replacement Row, err error)) ([]BatchOp, error) {
  1076. var ops []BatchOp
  1077. prefix := []byte(tx.sess.table.dataPrefix(table))
  1078. err := tx.base.pool.WithClient(func(client *KVClient) (retErr error) {
  1079. cursor, err := client.Scan(prefix)
  1080. if err != nil {
  1081. return err
  1082. }
  1083. defer func() {
  1084. if cerr := cursor.Close(); retErr == nil {
  1085. retErr = cerr
  1086. }
  1087. }()
  1088. for {
  1089. entries, done, err := cursor.Next()
  1090. if err != nil {
  1091. return err
  1092. }
  1093. for _, entry := range entries {
  1094. row, err := decodeRow(entry.Value)
  1095. if err != nil {
  1096. return err
  1097. }
  1098. write, replacement, err := transform(string(entry.Key), row)
  1099. if err != nil {
  1100. return err
  1101. }
  1102. if !write {
  1103. continue
  1104. }
  1105. if replacement == nil {
  1106. op := BatchOp{Op: batchDelete, Key: append([]byte(nil), entry.Key...)}
  1107. if err := budget.reserve(table, 1, batchOpBytes(op)); err != nil {
  1108. return err
  1109. }
  1110. ops = append(ops, op)
  1111. continue
  1112. }
  1113. value, err := encodeRow(replacement)
  1114. if err != nil {
  1115. return err
  1116. }
  1117. op := BatchOp{Op: batchPut, Key: append([]byte(nil), entry.Key...), Value: value}
  1118. if err := budget.reserve(table, 1, batchOpBytes(op)); err != nil {
  1119. return err
  1120. }
  1121. ops = append(ops, op)
  1122. }
  1123. if done {
  1124. return nil
  1125. }
  1126. }
  1127. })
  1128. return ops, err
  1129. }
  1130. // touchedTables lists the lowercased table names whose caches the commit must
  1131. // invalidate, including the source names rows moved away from.
  1132. func (tx *SchemaTx) touchedTables() []string {
  1133. set := make(map[string]bool)
  1134. for lower, schema := range tx.staged {
  1135. set[lower] = true
  1136. if schema != nil {
  1137. set[strings.ToLower(schema.Name)] = true
  1138. if source, ok := tx.dataSources[lower]; ok && source != "" {
  1139. set[source] = true
  1140. }
  1141. }
  1142. }
  1143. for table := range tx.drops {
  1144. set[table] = true
  1145. }
  1146. for table := range tx.renames {
  1147. set[table] = true
  1148. }
  1149. for table := range tx.rewrites {
  1150. set[table] = true
  1151. if source := tx.durableSource(table); source != "" {
  1152. set[source] = true
  1153. }
  1154. }
  1155. names := make([]string, 0, len(set))
  1156. for table := range set {
  1157. names = append(names, table)
  1158. }
  1159. sort.Strings(names)
  1160. return names
  1161. }
  1162. // touchedIndexes lists the lowercased index names whose caches the commit must
  1163. // reset.
  1164. func (tx *SchemaTx) touchedIndexes() []string {
  1165. names := make([]string, 0, len(tx.stagedIndexes))
  1166. for lower := range tx.stagedIndexes {
  1167. names = append(names, lower)
  1168. }
  1169. sort.Strings(names)
  1170. return names
  1171. }
  1172. // hasDataEffects reports whether the commit must rewrite or remove durable rows.
  1173. func (tx *SchemaTx) hasDataEffects() bool {
  1174. return len(tx.drops) > 0 || len(tx.renames) > 0 || len(tx.rewrites) > 0
  1175. }
  1176. // hasDDL reports whether the transaction staged a catalog change that the
  1177. // commit has to publish.
  1178. func (tx *SchemaTx) hasDDL() bool {
  1179. return len(tx.staged) > 0 || len(tx.stagedIndexes) > 0 || tx.catalogDirty || tx.indexListDirty || tx.hasDataEffects()
  1180. }
  1181. // Session catalog access
  1182. //
  1183. // A Session is the catalog view of one SQL connection. Outside an explicit
  1184. // transaction every accessor delegates to the shared SchemaManager; while a
  1185. // transaction is open the accessors resolve against its SchemaTx overlay, so
  1186. // the session reads its own DDL without publishing it.
  1187. // GetSchema returns the table definition this session observes.
  1188. func (s *Session) GetSchema(name string) (*Schema, error) {
  1189. s.mu.Lock()
  1190. defer s.mu.Unlock()
  1191. return s.getSchemaLocked(name)
  1192. }
  1193. // getSchemaLocked resolves a table definition (caller holds the session lock).
  1194. func (s *Session) getSchemaLocked(name string) (*Schema, error) {
  1195. if s.schemaTx != nil {
  1196. return s.schemaTx.getSchema(name)
  1197. }
  1198. return s.schema.GetSchema(name)
  1199. }
  1200. // TableExists reports whether the session can resolve a table.
  1201. func (s *Session) TableExists(name string) bool {
  1202. _, err := s.GetSchema(name)
  1203. return err == nil
  1204. }
  1205. // ListTables returns the table catalog this session observes.
  1206. func (s *Session) ListTables() ([]string, error) {
  1207. s.mu.Lock()
  1208. defer s.mu.Unlock()
  1209. if s.schemaTx != nil {
  1210. return s.schemaTx.listTables()
  1211. }
  1212. return s.schema.ListTables()
  1213. }
  1214. // GetIndex returns an index definition this session observes.
  1215. func (s *Session) GetIndex(name string) (*Index, error) {
  1216. s.mu.Lock()
  1217. defer s.mu.Unlock()
  1218. if s.schemaTx != nil {
  1219. return s.schemaTx.getIndex(name)
  1220. }
  1221. return s.schema.GetIndex(name)
  1222. }
  1223. // IndexExists reports whether the session can resolve an index.
  1224. func (s *Session) IndexExists(name string) bool {
  1225. _, err := s.GetIndex(name)
  1226. return err == nil
  1227. }
  1228. // ListIndexes returns the index catalog this session observes.
  1229. func (s *Session) ListIndexes() ([]string, error) {
  1230. s.mu.Lock()
  1231. defer s.mu.Unlock()
  1232. if s.schemaTx != nil {
  1233. return s.schemaTx.listIndexes()
  1234. }
  1235. return s.schema.ListIndexes()
  1236. }
  1237. // ListTableIndexes returns the index definitions of a table as this session
  1238. // observes them.
  1239. func (s *Session) ListTableIndexes(table string) ([]*Index, error) {
  1240. s.mu.Lock()
  1241. defer s.mu.Unlock()
  1242. if s.schemaTx != nil {
  1243. return s.schemaTx.listTableIndexes(table)
  1244. }
  1245. return s.schema.ListTableIndexes(table)
  1246. }
  1247. // Version identifies the catalog generation this session observes, including
  1248. // staged DDL, so the executor resynchronizes its analyzer catalog after a
  1249. // transactional DDL statement and again after the overlay is discarded.
  1250. func (s *Session) Version() uint64 {
  1251. s.mu.Lock()
  1252. defer s.mu.Unlock()
  1253. return s.versionLocked()
  1254. }
  1255. func (s *Session) versionLocked() uint64 {
  1256. if s.schemaTx != nil {
  1257. return s.schemaTx.version
  1258. }
  1259. return s.schema.Version()
  1260. }
  1261. // GetNextRowID allocates from the shared per-table ROWID counter. The counter is
  1262. // derived from durable rows and is not part of the catalog, so a transaction
  1263. // uses the same allocator as autocommit.
  1264. func (s *Session) GetNextRowID(table string) (int64, error) {
  1265. return s.schema.GetNextRowID(table)
  1266. }
  1267. // UpdateMaxRowID raises the shared per-table ROWID counter.
  1268. func (s *Session) UpdateMaxRowID(table string, rowid int64) error {
  1269. return s.schema.UpdateMaxRowID(table, rowid)
  1270. }
  1271. // CreateTable stages CREATE TABLE inside a transaction and publishes it
  1272. // immediately otherwise.
  1273. func (s *Session) CreateTable(schema *Schema) error {
  1274. s.mu.Lock()
  1275. defer s.mu.Unlock()
  1276. if s.schemaTx == nil {
  1277. return s.schema.CreateTable(schema)
  1278. }
  1279. return s.stageDDL(func(tx *SchemaTx) error { return tx.createTable(schema) })
  1280. }
  1281. // DropTable stages DROP TABLE inside a transaction and publishes it otherwise.
  1282. func (s *Session) DropTable(name string) error {
  1283. s.mu.Lock()
  1284. defer s.mu.Unlock()
  1285. if s.schemaTx == nil {
  1286. return s.guardTable(name, func() error { return s.schema.DropTable(name) })
  1287. }
  1288. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropTable(name) })
  1289. }
  1290. // CreateIndex stages CREATE INDEX inside a transaction and publishes it
  1291. // otherwise. A UNIQUE index is validated against the rows the session observes,
  1292. // so a transaction that staged duplicate rows fails the statement, not the
  1293. // commit.
  1294. func (s *Session) CreateIndex(index *Index) error {
  1295. s.mu.Lock()
  1296. defer s.mu.Unlock()
  1297. if s.schemaTx == nil {
  1298. return s.guardTable(index.Table, func() error {
  1299. if index.Unique {
  1300. return s.table.CreateUniqueIndex(index)
  1301. }
  1302. return s.schema.CreateIndex(index)
  1303. })
  1304. }
  1305. return s.stageDDL(func(tx *SchemaTx) error {
  1306. if err := tx.createIndex(index); err != nil {
  1307. return err
  1308. }
  1309. if !index.Unique {
  1310. return nil
  1311. }
  1312. return s.validateUniqueIndexLocked(index)
  1313. })
  1314. }
  1315. // validateUniqueIndexLocked rejects a staged UNIQUE index whose indexed values
  1316. // already collide in the rows this session observes (caller holds the session
  1317. // lock).
  1318. func (s *Session) validateUniqueIndexLocked(index *Index) error {
  1319. rows, err := s.selectLocked(index.Table, nil)
  1320. if err != nil {
  1321. return err
  1322. }
  1323. seen := make(map[string]struct{}, len(rows))
  1324. for _, row := range rows {
  1325. encoded, isNull, err := s.table.encodeUniqueValue(row, index.Columns)
  1326. if err != nil {
  1327. return err
  1328. }
  1329. if isNull {
  1330. continue
  1331. }
  1332. if _, duplicate := seen[encoded]; duplicate {
  1333. return &UniqueViolationError{Constraint: index.Name}
  1334. }
  1335. seen[encoded] = struct{}{}
  1336. }
  1337. return nil
  1338. }
  1339. // DropIndex stages DROP INDEX inside a transaction and publishes it otherwise.
  1340. func (s *Session) DropIndex(name string) error {
  1341. s.mu.Lock()
  1342. defer s.mu.Unlock()
  1343. if s.schemaTx == nil {
  1344. return s.schema.DropIndex(name)
  1345. }
  1346. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropIndex(name) })
  1347. }
  1348. // AddColumn stages ALTER TABLE ADD COLUMN inside a transaction and publishes it
  1349. // otherwise.
  1350. func (s *Session) AddColumn(table string, column Column, checks []CheckConstraint, unique *Index) error {
  1351. s.mu.Lock()
  1352. defer s.mu.Unlock()
  1353. if s.schemaTx == nil {
  1354. return s.guardTable(table, func() error { return s.schema.AddColumn(table, column, checks, unique) })
  1355. }
  1356. return s.stageDDL(func(tx *SchemaTx) error { return tx.addColumn(table, column, checks, unique) })
  1357. }
  1358. // DropColumn stages ALTER TABLE DROP COLUMN inside a transaction and publishes
  1359. // it otherwise.
  1360. func (s *Session) DropColumn(table, columnName string) error {
  1361. s.mu.Lock()
  1362. defer s.mu.Unlock()
  1363. if s.schemaTx == nil {
  1364. return s.guardTable(table, func() error { return s.schema.DropColumn(table, columnName) })
  1365. }
  1366. return s.stageDDL(func(tx *SchemaTx) error { return tx.dropColumn(table, columnName) })
  1367. }
  1368. // RenameTable stages ALTER TABLE RENAME TABLE inside a transaction and
  1369. // publishes it otherwise.
  1370. func (s *Session) RenameTable(oldName, newName string) error {
  1371. s.mu.Lock()
  1372. defer s.mu.Unlock()
  1373. if s.schemaTx == nil {
  1374. return s.guardTable(oldName, func() error { return s.schema.RenameTable(oldName, newName) })
  1375. }
  1376. return s.stageDDL(func(tx *SchemaTx) error { return tx.renameTable(oldName, newName) })
  1377. }
  1378. // RenameColumn stages ALTER TABLE RENAME COLUMN inside a transaction and
  1379. // publishes it otherwise.
  1380. func (s *Session) RenameColumn(table, oldName, newName string) error {
  1381. s.mu.Lock()
  1382. defer s.mu.Unlock()
  1383. if s.schemaTx == nil {
  1384. return s.guardTable(table, func() error { return s.schema.RenameColumn(table, oldName, newName) })
  1385. }
  1386. return s.stageDDL(func(tx *SchemaTx) error { return tx.renameColumn(table, oldName, newName) })
  1387. }
  1388. // stageDDL runs one DDL statement against the transaction overlay and restores
  1389. // the catalog, overlay, and read set if it fails, so a rejected statement
  1390. // leaves the transaction exactly as it was (caller holds the session lock).
  1391. func (s *Session) stageDDL(apply func(tx *SchemaTx) error) error {
  1392. savepoint := s.savepointLocked()
  1393. if err := apply(s.schemaTx); err != nil {
  1394. s.rollbackToSavepointLocked(savepoint)
  1395. return err
  1396. }
  1397. return nil
  1398. }
  1399. // overlayHolds reports whether the session staged the given durable key of a
  1400. // table.
  1401. func (s *Session) overlayHolds(lowerTable, key string) bool {
  1402. _, ok := s.overlay[lowerTable][key]
  1403. return ok
  1404. }
  1405. // markTableScanned pins the table's current generation as a read so the commit
  1406. // fails with a serialization error if another session writes the table between
  1407. // the DDL statement and the commit.
  1408. func (s *Session) markTableScanned(table string) {
  1409. lower := strings.ToLower(table)
  1410. if _, ok := s.scanGens[lower]; !ok {
  1411. s.scanGens[lower] = s.table.generation(lower)
  1412. }
  1413. }
  1414. // dropOverlayTableLocked discards the staged rows of a table the transaction
  1415. // dropped, including their mutation-log entries (caller holds the session lock).
  1416. func (s *Session) dropOverlayTableLocked(lower string) {
  1417. delete(s.overlay, lower)
  1418. log := s.log[:0]
  1419. for _, mutation := range s.log {
  1420. if mutation.table != lower {
  1421. log = append(log, mutation)
  1422. }
  1423. }
  1424. s.log = log
  1425. delete(s.scanGens, lower)
  1426. for key, predicate := range s.predicateGens {
  1427. if predicate.table == lower {
  1428. delete(s.predicateGens, key)
  1429. }
  1430. }
  1431. prefix := s.table.dataPrefix(lower)
  1432. for key := range s.reads {
  1433. if strings.HasPrefix(key, prefix) {
  1434. delete(s.reads, key)
  1435. }
  1436. }
  1437. }
  1438. // rekeyOverlayLocked moves the staged rows of a renamed table to the new data
  1439. // prefix so the transaction reads its own rows back under the new name
  1440. // (caller holds the session lock).
  1441. func (s *Session) rekeyOverlayLocked(oldTable, newTable string) {
  1442. oldLower := strings.ToLower(oldTable)
  1443. newLower := strings.ToLower(newTable)
  1444. if oldLower == newLower {
  1445. return
  1446. }
  1447. oldPrefix := s.table.dataPrefix(oldLower)
  1448. newPrefix := s.table.dataPrefix(newLower)
  1449. if entries, ok := s.overlay[oldLower]; ok {
  1450. delete(s.overlay, oldLower)
  1451. moved := make(map[string]*overlayEntry, len(entries))
  1452. for key, entry := range entries {
  1453. moved[newPrefix+strings.TrimPrefix(key, oldPrefix)] = entry
  1454. }
  1455. s.overlay[newLower] = moved
  1456. }
  1457. for i := range s.log {
  1458. if s.log[i].table != oldLower {
  1459. continue
  1460. }
  1461. s.log[i].table = newLower
  1462. s.log[i].key = newPrefix + strings.TrimPrefix(s.log[i].key, oldPrefix)
  1463. }
  1464. if gen, ok := s.scanGens[oldLower]; ok {
  1465. delete(s.scanGens, oldLower)
  1466. s.scanGens[newLower] = gen
  1467. }
  1468. reads := make(map[string]uint64, len(s.reads))
  1469. for key, lsn := range s.reads {
  1470. if strings.HasPrefix(key, oldPrefix) {
  1471. reads[newPrefix+strings.TrimPrefix(key, oldPrefix)] = lsn
  1472. continue
  1473. }
  1474. reads[key] = lsn
  1475. }
  1476. s.reads = reads
  1477. predicates := make(map[string]predicateRead, len(s.predicateGens))
  1478. for key, predicate := range s.predicateGens {
  1479. if predicate.table == oldLower {
  1480. predicate.table = newLower
  1481. }
  1482. if strings.HasPrefix(key, oldPrefix) {
  1483. key = newPrefix + strings.TrimPrefix(key, oldPrefix)
  1484. }
  1485. predicates[key] = predicate
  1486. }
  1487. s.predicateGens = predicates
  1488. }
  1489. // ForeignKeyParticipants reports whether a write to table can affect referential
  1490. // integrity as this session observes the catalog.
  1491. func (s *Session) ForeignKeyParticipants(table string) (bool, error) {
  1492. s.mu.Lock()
  1493. defer s.mu.Unlock()
  1494. if s.schemaTx == nil {
  1495. return s.schema.ForeignKeyParticipants(table)
  1496. }
  1497. catalog, err := s.foreignKeyCatalogLocked()
  1498. if err != nil {
  1499. return false, err
  1500. }
  1501. return catalog.participants[strings.ToLower(table)], nil
  1502. }
  1503. // ForeignKeyColumnDependency returns the constraints that keep a column in
  1504. // place, as this session observes the catalog.
  1505. func (s *Session) ForeignKeyColumnDependency(table, column string) ([]string, error) {
  1506. s.mu.Lock()
  1507. defer s.mu.Unlock()
  1508. if s.schemaTx == nil {
  1509. return s.schema.ForeignKeyColumnDependency(table, column)
  1510. }
  1511. catalog, err := s.foreignKeyCatalogLocked()
  1512. if err != nil {
  1513. return nil, err
  1514. }
  1515. schema, err := s.getSchemaLocked(table)
  1516. if err != nil {
  1517. return nil, err
  1518. }
  1519. seen := make(map[string]bool)
  1520. var labels []string
  1521. add := func(child string, fk ForeignKeyConstraint) {
  1522. label := foreignKeyReferenceName(child, fk)
  1523. if seen[label] {
  1524. return
  1525. }
  1526. seen[label] = true
  1527. labels = append(labels, label)
  1528. }
  1529. for _, fk := range schema.ForeignKeys {
  1530. for _, name := range fk.Columns {
  1531. if strings.EqualFold(name, column) {
  1532. add(schema.Name, fk)
  1533. break
  1534. }
  1535. }
  1536. }
  1537. for _, ref := range catalog.referencing[strings.ToLower(table)] {
  1538. for _, name := range ref.FK.RefColumns {
  1539. if strings.EqualFold(name, column) {
  1540. add(ref.Table, ref.FK)
  1541. break
  1542. }
  1543. }
  1544. }
  1545. sort.Strings(labels)
  1546. return labels, nil
  1547. }
  1548. // ForeignKeyTableDependency returns the constraints that name table as their
  1549. // referenced parent, as this session observes the catalog.
  1550. func (s *Session) ForeignKeyTableDependency(table string) ([]string, error) {
  1551. s.mu.Lock()
  1552. defer s.mu.Unlock()
  1553. if s.schemaTx == nil {
  1554. return s.schema.ForeignKeyTableDependency(table)
  1555. }
  1556. catalog, err := s.foreignKeyCatalogLocked()
  1557. if err != nil {
  1558. return nil, err
  1559. }
  1560. seen := make(map[string]bool)
  1561. var labels []string
  1562. for _, ref := range catalog.referencing[strings.ToLower(table)] {
  1563. label := foreignKeyReferenceName(ref.Table, ref.FK)
  1564. if seen[label] {
  1565. continue
  1566. }
  1567. seen[label] = true
  1568. labels = append(labels, label)
  1569. }
  1570. sort.Strings(labels)
  1571. return labels, nil
  1572. }
  1573. // ForeignKeyClosure returns the constrained component of the given tables as
  1574. // this session observes the catalog.
  1575. func (s *Session) ForeignKeyClosure(tables []string) (map[string]bool, error) {
  1576. s.mu.Lock()
  1577. defer s.mu.Unlock()
  1578. if s.schemaTx == nil {
  1579. return s.schema.ForeignKeyClosure(tables)
  1580. }
  1581. catalog, err := s.foreignKeyCatalogLocked()
  1582. if err != nil {
  1583. return nil, err
  1584. }
  1585. return catalog.closure(tables), nil
  1586. }
  1587. // foreignKeyCatalogLocked builds the foreign-key metadata of the session's
  1588. // catalog view (caller holds the session lock).
  1589. func (s *Session) foreignKeyCatalogLocked() (*foreignKeyCatalog, error) {
  1590. version := s.versionLocked()
  1591. if s.fkCatalog != nil && s.fkVersion == version {
  1592. return s.fkCatalog, nil
  1593. }
  1594. tables, err := s.schemaTx.listTables()
  1595. if err != nil {
  1596. return nil, err
  1597. }
  1598. catalog, err := buildForeignKeyCatalog(version, tables, func(name string) (*Schema, error) {
  1599. return s.getSchemaLocked(name)
  1600. })
  1601. if err != nil {
  1602. return nil, err
  1603. }
  1604. s.fkCatalog = catalog
  1605. s.fkVersion = version
  1606. return catalog, nil
  1607. }