schema_test.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872
  1. package storage
  2. import (
  3. "bufio"
  4. "errors"
  5. "fmt"
  6. "net"
  7. "sort"
  8. "strings"
  9. "sync"
  10. "testing"
  11. "time"
  12. )
  13. type testScan struct {
  14. keys []string
  15. offset int
  16. limit uint32
  17. keysOnly bool
  18. }
  19. type testKVServer struct {
  20. mu sync.Mutex
  21. data map[string][]byte
  22. lsns map[string]uint64
  23. writes map[string]int
  24. nextLSN uint64
  25. maxScanPage uint32
  26. scanOpens int
  27. scanNexts int
  28. scanCloses int
  29. keyOnlyOpens int
  30. gets int
  31. multiGets int
  32. closers []net.Conn
  33. }
  34. func newTestKVServer(t *testing.T) *testKVServer {
  35. t.Helper()
  36. return &testKVServer{
  37. data: make(map[string][]byte),
  38. lsns: make(map[string]uint64),
  39. writes: make(map[string]int),
  40. }
  41. }
  42. func newTestKVPool(kv *testKVServer, size int, timeout time.Duration) *KVPool {
  43. pool := &KVPool{
  44. pool: make(chan *KVClient, size),
  45. size: size,
  46. timeout: timeout,
  47. }
  48. for i := 0; i < size; i++ {
  49. pool.pool <- kv.client()
  50. }
  51. return pool
  52. }
  53. func (s *testKVServer) close() {
  54. s.mu.Lock()
  55. closers := append([]net.Conn(nil), s.closers...)
  56. s.mu.Unlock()
  57. for _, conn := range closers {
  58. _ = conn.Close()
  59. }
  60. }
  61. func (s *testKVServer) client() *KVClient {
  62. clientConn, serverConn := net.Pipe()
  63. s.mu.Lock()
  64. s.closers = append(s.closers, clientConn, serverConn)
  65. s.mu.Unlock()
  66. go s.handle(serverConn)
  67. return &KVClient{
  68. conn: clientConn,
  69. reader: bufio.NewReader(clientConn),
  70. writer: bufio.NewWriter(clientConn),
  71. nextID: 1,
  72. lastUsed: time.Now(),
  73. }
  74. }
  75. func (s *testKVServer) writeCount(prefix string) int {
  76. s.mu.Lock()
  77. defer s.mu.Unlock()
  78. var count int
  79. for key, writes := range s.writes {
  80. if strings.Contains(key, prefix) {
  81. count += writes
  82. }
  83. }
  84. return count
  85. }
  86. func (s *testKVServer) hasKey(key string) bool {
  87. s.mu.Lock()
  88. defer s.mu.Unlock()
  89. _, ok := s.data[key]
  90. return ok
  91. }
  92. func (s *testKVServer) countKeys(prefix string) int {
  93. s.mu.Lock()
  94. defer s.mu.Unlock()
  95. var count int
  96. for key := range s.data {
  97. if strings.HasPrefix(key, prefix) {
  98. count++
  99. }
  100. }
  101. return count
  102. }
  103. func (s *testKVServer) scanStats() (opens, nexts, closes int) {
  104. s.mu.Lock()
  105. defer s.mu.Unlock()
  106. return s.scanOpens, s.scanNexts, s.scanCloses
  107. }
  108. func (s *testKVServer) keyOnlyOpenCount() int {
  109. s.mu.Lock()
  110. defer s.mu.Unlock()
  111. return s.keyOnlyOpens
  112. }
  113. func (s *testKVServer) readStats() (gets, multiGets int) {
  114. s.mu.Lock()
  115. defer s.mu.Unlock()
  116. return s.gets, s.multiGets
  117. }
  118. func (s *testKVServer) handle(conn net.Conn) {
  119. defer conn.Close()
  120. r := bufio.NewReader(conn)
  121. scans := make(map[uint64]*testScan)
  122. var nextScan uint64 = 1
  123. for {
  124. opcode, _, requestID, payload, err := readFrame(r)
  125. if err != nil {
  126. return
  127. }
  128. body := s.execute(opcode, payload, scans, &nextScan)
  129. if _, err := conn.Write(encodeResponse(opcode, requestID, body)); err != nil {
  130. return
  131. }
  132. }
  133. }
  134. func (s *testKVServer) execute(opcode uint16, payload []byte, scans map[uint64]*testScan, nextScan *uint64) []byte {
  135. switch opcode {
  136. case opPing:
  137. body := make([]byte, 2+len(payload))
  138. putU16(body[0:2], statusOK)
  139. copy(body[2:], payload)
  140. return body
  141. case opGet:
  142. key, ok := parseOneKey(payload)
  143. if !ok {
  144. return errorBody("InvalidPayload")
  145. }
  146. s.mu.Lock()
  147. s.gets++
  148. value, found := s.data[string(key)]
  149. lsn := s.lsns[string(key)]
  150. s.mu.Unlock()
  151. if !found {
  152. body := make([]byte, 2)
  153. putU16(body[0:2], statusNotFound)
  154. return body
  155. }
  156. body := make([]byte, 14+len(value))
  157. putU16(body[0:2], statusOK)
  158. putU64(body[2:10], lsn)
  159. putU32(body[10:14], uint32(len(value)))
  160. copy(body[14:], value)
  161. return body
  162. case opPut:
  163. key, value, ok := parsePut(payload)
  164. if !ok {
  165. return errorBody("InvalidPayload")
  166. }
  167. s.mu.Lock()
  168. s.nextLSN++
  169. lsn := s.nextLSN
  170. s.data[string(key)] = append([]byte(nil), value...)
  171. s.lsns[string(key)] = lsn
  172. s.writes[string(key)]++
  173. s.mu.Unlock()
  174. body := make([]byte, 10)
  175. putU16(body[0:2], statusOK)
  176. putU64(body[2:10], lsn)
  177. return body
  178. case opDelete:
  179. key, ok := parseOneKey(payload)
  180. if !ok {
  181. return errorBody("InvalidPayload")
  182. }
  183. s.mu.Lock()
  184. _, found := s.data[string(key)]
  185. delete(s.data, string(key))
  186. delete(s.lsns, string(key))
  187. s.mu.Unlock()
  188. body := make([]byte, 3)
  189. putU16(body[0:2], statusOK)
  190. if found {
  191. body[2] = 1
  192. }
  193. return body
  194. case opExists:
  195. key, ok := parseOneKey(payload)
  196. if !ok {
  197. return errorBody("InvalidPayload")
  198. }
  199. s.mu.Lock()
  200. _, found := s.data[string(key)]
  201. s.mu.Unlock()
  202. body := make([]byte, 3)
  203. putU16(body[0:2], statusOK)
  204. if found {
  205. body[2] = 1
  206. }
  207. return body
  208. case opMultiGet:
  209. keys, ok := parseMultiGetKeys(payload)
  210. if !ok {
  211. return errorBody("InvalidPayload")
  212. }
  213. body := make([]byte, 6)
  214. putU16(body[0:2], statusOK)
  215. putU32(body[2:6], uint32(len(keys)))
  216. s.mu.Lock()
  217. s.multiGets++
  218. for _, key := range keys {
  219. value, found := s.data[string(key)]
  220. if !found {
  221. body = append(body, make([]byte, 16)...)
  222. continue
  223. }
  224. lsn := s.lsns[string(key)]
  225. entry := make([]byte, 16+len(value))
  226. entry[0] = 1
  227. putU32(entry[4:8], uint32(len(value)))
  228. putU64(entry[8:16], lsn)
  229. copy(entry[16:], value)
  230. body = append(body, entry...)
  231. }
  232. s.mu.Unlock()
  233. return body
  234. case opBatchWrite:
  235. ops, ok := parseBatchOps(payload)
  236. if !ok {
  237. return errorBody("InvalidPayload")
  238. }
  239. s.mu.Lock()
  240. s.nextLSN++
  241. lsn := s.nextLSN
  242. for _, op := range ops {
  243. if op.Op == batchPut {
  244. s.data[string(op.Key)] = append([]byte(nil), op.Value...)
  245. s.lsns[string(op.Key)] = lsn
  246. s.writes[string(op.Key)]++
  247. } else {
  248. delete(s.data, string(op.Key))
  249. delete(s.lsns, string(op.Key))
  250. }
  251. }
  252. s.mu.Unlock()
  253. body := make([]byte, 10)
  254. putU16(body[0:2], statusOK)
  255. putU64(body[2:10], lsn)
  256. return body
  257. case opCompareBatch:
  258. checks, ops, ok := parseCompareBatch(payload)
  259. if !ok {
  260. return errorBody("InvalidPayload")
  261. }
  262. s.mu.Lock()
  263. committed := true
  264. for _, check := range checks {
  265. if check.LSN == 0 {
  266. if _, found := s.data[string(check.Key)]; found {
  267. committed = false
  268. break
  269. }
  270. } else if s.lsns[string(check.Key)] != check.LSN {
  271. committed = false
  272. break
  273. }
  274. }
  275. var lsn uint64
  276. if committed {
  277. s.nextLSN++
  278. lsn = s.nextLSN
  279. for _, op := range ops {
  280. if op.Op == batchPut {
  281. s.data[string(op.Key)] = append([]byte(nil), op.Value...)
  282. s.lsns[string(op.Key)] = lsn
  283. s.writes[string(op.Key)]++
  284. } else {
  285. delete(s.data, string(op.Key))
  286. delete(s.lsns, string(op.Key))
  287. }
  288. }
  289. }
  290. s.mu.Unlock()
  291. body := make([]byte, 18)
  292. putU16(body[0:2], statusOK)
  293. if committed {
  294. body[2] = 1
  295. }
  296. putU64(body[10:18], lsn)
  297. return body
  298. case opScanOpen:
  299. includeValues, limit, prefix, ok := parseScanOpen(payload)
  300. if !ok {
  301. return errorBody("InvalidPayload")
  302. }
  303. s.mu.Lock()
  304. s.scanOpens++
  305. if !includeValues {
  306. s.keyOnlyOpens++
  307. }
  308. s.mu.Unlock()
  309. s.mu.Lock()
  310. keys := make([]string, 0)
  311. for key := range s.data {
  312. if strings.HasPrefix(key, string(prefix)) {
  313. keys = append(keys, key)
  314. }
  315. }
  316. s.mu.Unlock()
  317. sort.Strings(keys)
  318. if s.maxScanPage > 0 && limit > s.maxScanPage {
  319. limit = s.maxScanPage
  320. }
  321. id := *nextScan
  322. *nextScan = id + 1
  323. scans[id] = &testScan{keys: keys, limit: limit, keysOnly: !includeValues}
  324. body := make([]byte, 10)
  325. putU16(body[0:2], statusOK)
  326. putU64(body[2:10], id)
  327. return body
  328. case opScanNext:
  329. id, ok := parseScanID(payload)
  330. if !ok {
  331. return errorBody("InvalidPayload")
  332. }
  333. scan := scans[id]
  334. if scan == nil {
  335. return errorBody("ScanNotFound")
  336. }
  337. s.mu.Lock()
  338. s.scanNexts++
  339. s.mu.Unlock()
  340. remaining := len(scan.keys) - scan.offset
  341. count := int(scan.limit)
  342. if count > remaining {
  343. count = remaining
  344. }
  345. end := scan.offset + count
  346. body := make([]byte, 10)
  347. putU16(body[0:2], statusOK)
  348. if end >= len(scan.keys) {
  349. body[2] = 1
  350. }
  351. putU32(body[6:10], uint32(count))
  352. s.mu.Lock()
  353. for _, key := range scan.keys[scan.offset:end] {
  354. value := s.data[key]
  355. if scan.keysOnly {
  356. value = nil
  357. }
  358. lsn := s.lsns[key]
  359. entry := make([]byte, 16+len(key)+len(value))
  360. putU32(entry[0:4], uint32(len(key)))
  361. putU32(entry[4:8], uint32(len(value)))
  362. putU64(entry[8:16], lsn)
  363. copy(entry[16:], key)
  364. copy(entry[16+len(key):], value)
  365. body = append(body, entry...)
  366. }
  367. s.mu.Unlock()
  368. scan.offset = end
  369. return body
  370. case opScanClose:
  371. id, ok := parseScanID(payload)
  372. if !ok {
  373. return errorBody("InvalidPayload")
  374. }
  375. s.mu.Lock()
  376. s.scanCloses++
  377. s.mu.Unlock()
  378. delete(scans, id)
  379. body := make([]byte, 3)
  380. putU16(body[0:2], statusOK)
  381. body[2] = 1
  382. return body
  383. default:
  384. return errorBody("UnknownOpcode")
  385. }
  386. }
  387. func parsePut(payload []byte) ([]byte, []byte, bool) {
  388. if len(payload) < 8 {
  389. return nil, nil, false
  390. }
  391. keyLen := getU32(payload[0:4])
  392. valueLen := getU32(payload[4:8])
  393. if keyLen > maxKeySize || valueLen > maxValueSize {
  394. return nil, nil, false
  395. }
  396. if uint64(8)+uint64(keyLen)+uint64(valueLen) != uint64(len(payload)) {
  397. return nil, nil, false
  398. }
  399. return payload[8 : 8+keyLen], payload[8+keyLen:], true
  400. }
  401. func parseMultiGetKeys(payload []byte) ([][]byte, bool) {
  402. if len(payload) < 4 {
  403. return nil, false
  404. }
  405. count := getU32(payload[0:4])
  406. if count > maxOperations {
  407. return nil, false
  408. }
  409. keys := make([][]byte, 0, count)
  410. pos := 4
  411. for i := uint32(0); i < count; i++ {
  412. if len(payload)-pos < 4 {
  413. return nil, false
  414. }
  415. length := getU32(payload[pos : pos+4])
  416. pos += 4
  417. if length > maxKeySize || len(payload)-pos < int(length) {
  418. return nil, false
  419. }
  420. keys = append(keys, payload[pos:pos+int(length)])
  421. pos += int(length)
  422. }
  423. return keys, pos == len(payload)
  424. }
  425. func parseBatchOps(payload []byte) ([]BatchOp, bool) {
  426. if len(payload) < 8 {
  427. return nil, false
  428. }
  429. count := getU32(payload[0:4])
  430. metadataLen := getU32(payload[4:8])
  431. if count == 0 || count > maxOperations || uint64(metadataLen) > uint64(len(payload)-8) {
  432. return nil, false
  433. }
  434. pos := 8 + int(metadataLen)
  435. ops := make([]BatchOp, 0, count)
  436. for i := uint32(0); i < count; i++ {
  437. if len(payload)-pos < 12 {
  438. return nil, false
  439. }
  440. opcode := payload[pos]
  441. keyLen := getU32(payload[pos+4 : pos+8])
  442. valueLen := getU32(payload[pos+8 : pos+12])
  443. pos += 12
  444. if opcode != batchPut && opcode != batchDelete {
  445. return nil, false
  446. }
  447. if keyLen > maxKeySize || valueLen > maxValueSize {
  448. return nil, false
  449. }
  450. if opcode == batchDelete && valueLen != 0 {
  451. return nil, false
  452. }
  453. if len(payload)-pos < int(keyLen)+int(valueLen) {
  454. return nil, false
  455. }
  456. key := payload[pos : pos+int(keyLen)]
  457. pos += int(keyLen)
  458. value := payload[pos : pos+int(valueLen)]
  459. pos += int(valueLen)
  460. ops = append(ops, BatchOp{Op: opcode, Key: key, Value: value})
  461. }
  462. return ops, pos == len(payload)
  463. }
  464. func parseCompareBatch(payload []byte) ([]CompareCheck, []BatchOp, bool) {
  465. if len(payload) < 16 {
  466. return nil, nil, false
  467. }
  468. numChecks := getU32(payload[0:4])
  469. numOps := getU32(payload[4:8])
  470. metadataLen := getU32(payload[8:12])
  471. if numChecks > maxOperations || numOps == 0 || numOps > maxOperations {
  472. return nil, nil, false
  473. }
  474. if uint64(16)+uint64(metadataLen) > uint64(len(payload)) {
  475. return nil, nil, false
  476. }
  477. pos := 16
  478. checks := make([]CompareCheck, 0, numChecks)
  479. for i := uint32(0); i < numChecks; i++ {
  480. if len(payload)-pos < 16 {
  481. return nil, nil, false
  482. }
  483. keyLen := getU32(payload[pos : pos+4])
  484. lsn := getU64(payload[pos+8 : pos+16])
  485. pos += 16
  486. if keyLen > maxKeySize || len(payload)-pos < int(keyLen) {
  487. return nil, nil, false
  488. }
  489. checks = append(checks, CompareCheck{Key: payload[pos : pos+int(keyLen)], LSN: lsn})
  490. pos += int(keyLen)
  491. }
  492. pos += int(metadataLen)
  493. if pos > len(payload) {
  494. return nil, nil, false
  495. }
  496. ops := make([]BatchOp, 0, numOps)
  497. for i := uint32(0); i < numOps; i++ {
  498. if len(payload)-pos < 12 {
  499. return nil, nil, false
  500. }
  501. opcode := payload[pos]
  502. keyLen := getU32(payload[pos+4 : pos+8])
  503. valueLen := getU32(payload[pos+8 : pos+12])
  504. pos += 12
  505. if opcode != batchPut && opcode != batchDelete {
  506. return nil, nil, false
  507. }
  508. if keyLen > maxKeySize || valueLen > maxValueSize {
  509. return nil, nil, false
  510. }
  511. if opcode == batchDelete && valueLen != 0 {
  512. return nil, nil, false
  513. }
  514. if len(payload)-pos < int(keyLen)+int(valueLen) {
  515. return nil, nil, false
  516. }
  517. key := payload[pos : pos+int(keyLen)]
  518. pos += int(keyLen)
  519. value := payload[pos : pos+int(valueLen)]
  520. pos += int(valueLen)
  521. ops = append(ops, BatchOp{Op: opcode, Key: key, Value: value})
  522. }
  523. return checks, ops, pos == len(payload)
  524. }
  525. func parseScanOpen(payload []byte) (bool, uint32, []byte, bool) {
  526. if len(payload) < 12 {
  527. return false, 0, nil, false
  528. }
  529. includeValues := payload[0] != 0
  530. limit := getU32(payload[4:8])
  531. prefixLen := getU32(payload[8:12])
  532. if limit == 0 || limit > 4096 || prefixLen > maxKeySize {
  533. return false, 0, nil, false
  534. }
  535. if uint64(12)+uint64(prefixLen) != uint64(len(payload)) {
  536. return false, 0, nil, false
  537. }
  538. return includeValues, limit, payload[12:], true
  539. }
  540. func parseScanID(payload []byte) (uint64, bool) {
  541. if len(payload) < 8 {
  542. return 0, false
  543. }
  544. return getU64(payload[0:8]), true
  545. }
  546. func TestInsertDoesNotRewriteSchemaForRowIDUpdates(t *testing.T) {
  547. kv := newTestKVServer(t)
  548. defer kv.close()
  549. pool := newTestKVPool(kv, 2, 5*time.Second)
  550. defer pool.Close()
  551. schemas := NewSchemaManager(pool, "testdb")
  552. tables := NewTableManager(pool, schemas, "testdb")
  553. err := schemas.CreateTable(&Schema{
  554. Name: "users",
  555. Columns: []Column{
  556. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  557. {Name: "name", Type: "TEXT", Nullable: true},
  558. },
  559. })
  560. if err != nil {
  561. t.Fatalf("create table: %v", err)
  562. }
  563. initialSchemaWrites := kv.writeCount(":_schema:")
  564. if initialSchemaWrites != 1 {
  565. t.Fatalf("expected create table to write schema once, got %d", initialSchemaWrites)
  566. }
  567. for i := int64(1); i <= 3; i++ {
  568. err := tables.Insert("users", Row{"id": i, "name": fmt.Sprintf("user-%d", i)})
  569. if err != nil {
  570. t.Fatalf("insert %d: %v", i, err)
  571. }
  572. }
  573. if got := kv.writeCount(":_schema:"); got != initialSchemaWrites {
  574. t.Fatalf("expected inserts not to rewrite schema, got %d schema writes", got)
  575. }
  576. if got := kv.writeCount(":_sys:rowid:"); got != 0 {
  577. t.Fatalf("expected no rowid counter writes for inserts, got %d", got)
  578. }
  579. }
  580. func TestRowIDIsDerivedFromRowsAfterRestart(t *testing.T) {
  581. kv := newTestKVServer(t)
  582. defer kv.close()
  583. pool := newTestKVPool(kv, 2, 5*time.Second)
  584. defer pool.Close()
  585. schemas := NewSchemaManager(pool, "testdb")
  586. tables := NewTableManager(pool, schemas, "testdb")
  587. err := schemas.CreateTable(&Schema{
  588. Name: "events",
  589. Columns: []Column{
  590. {Name: "name", Type: "TEXT", Nullable: true},
  591. },
  592. })
  593. if err != nil {
  594. t.Fatalf("create table: %v", err)
  595. }
  596. for i := 1; i <= 2; i++ {
  597. err := tables.Insert("events", Row{"name": fmt.Sprintf("event-%d", i)})
  598. if err != nil {
  599. t.Fatalf("insert %d: %v", i, err)
  600. }
  601. }
  602. // Simulate a process restart: new managers have empty in-memory ROWID state
  603. // but the same durable KV rows.
  604. restartedSchemas := NewSchemaManager(pool, "testdb")
  605. restartedTables := NewTableManager(pool, restartedSchemas, "testdb")
  606. if err := restartedTables.Insert("events", Row{"name": "event-3"}); err != nil {
  607. t.Fatalf("insert after restart: %v", err)
  608. }
  609. if !kv.hasKey("testdb:_data:events:3") {
  610. t.Fatalf("expected restart insert to continue at rowid 3")
  611. }
  612. if got := kv.writeCount(":_sys:rowid:"); got != 0 {
  613. t.Fatalf("expected no rowid counter writes, got %d", got)
  614. }
  615. }
  616. func TestInsertDoesNotWriteDurableIndexEntries(t *testing.T) {
  617. kv := newTestKVServer(t)
  618. defer kv.close()
  619. pool := newTestKVPool(kv, 2, 5*time.Second)
  620. defer pool.Close()
  621. schemas := NewSchemaManager(pool, "testdb")
  622. tables := NewTableManager(pool, schemas, "testdb")
  623. err := schemas.CreateTable(&Schema{
  624. Name: "users",
  625. Columns: []Column{
  626. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  627. {Name: "status", Type: "TEXT", Nullable: false},
  628. },
  629. })
  630. if err != nil {
  631. t.Fatalf("create table: %v", err)
  632. }
  633. err = schemas.CreateIndex(&Index{
  634. Name: "idx_users_status",
  635. Table: "users",
  636. Columns: []IndexColumn{
  637. {Name: "status"},
  638. },
  639. })
  640. if err != nil {
  641. t.Fatalf("create index: %v", err)
  642. }
  643. for i := int64(1); i <= 3; i++ {
  644. status := "active"
  645. if i == 2 {
  646. status = "inactive"
  647. }
  648. err := tables.Insert("users", Row{"id": i, "status": status})
  649. if err != nil {
  650. t.Fatalf("insert %d: %v", i, err)
  651. }
  652. }
  653. if got := kv.writeCount(":idx:"); got != 0 {
  654. t.Fatalf("expected no durable index entry writes, got %d", got)
  655. }
  656. rows, err := tables.SelectByIndex("users", "idx_users_status", "active")
  657. if err != nil {
  658. t.Fatalf("select by index: %v", err)
  659. }
  660. if len(rows) != 2 {
  661. t.Fatalf("expected 2 active rows from derived index, got %d", len(rows))
  662. }
  663. }
  664. func TestIndexIsDerivedFromRowsAfterRestart(t *testing.T) {
  665. kv := newTestKVServer(t)
  666. defer kv.close()
  667. pool := newTestKVPool(kv, 2, 5*time.Second)
  668. defer pool.Close()
  669. schemas := NewSchemaManager(pool, "testdb")
  670. tables := NewTableManager(pool, schemas, "testdb")
  671. err := schemas.CreateTable(&Schema{
  672. Name: "users",
  673. Columns: []Column{
  674. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  675. {Name: "status", Type: "TEXT", Nullable: false},
  676. },
  677. })
  678. if err != nil {
  679. t.Fatalf("create table: %v", err)
  680. }
  681. err = schemas.CreateIndex(&Index{
  682. Name: "idx_users_status",
  683. Table: "users",
  684. Columns: []IndexColumn{
  685. {Name: "status"},
  686. },
  687. })
  688. if err != nil {
  689. t.Fatalf("create index: %v", err)
  690. }
  691. for i := int64(1); i <= 3; i++ {
  692. status := "active"
  693. if i == 3 {
  694. status = "inactive"
  695. }
  696. if err := tables.Insert("users", Row{"id": i, "status": status}); err != nil {
  697. t.Fatalf("insert %d: %v", i, err)
  698. }
  699. }
  700. restartedSchemas := NewSchemaManager(pool, "testdb")
  701. restartedTables := NewTableManager(pool, restartedSchemas, "testdb")
  702. rows, err := restartedTables.SelectByIndex("users", "idx_users_status", "active")
  703. if err != nil {
  704. t.Fatalf("select by index after restart: %v", err)
  705. }
  706. if len(rows) != 2 {
  707. t.Fatalf("expected 2 active rows from restart-derived index, got %d", len(rows))
  708. }
  709. if got := kv.writeCount(":idx:"); got != 0 {
  710. t.Fatalf("expected no durable index entry writes, got %d", got)
  711. }
  712. }
  713. func TestListTableIndexesCachesMetadata(t *testing.T) {
  714. kv := newTestKVServer(t)
  715. defer kv.close()
  716. pool := newTestKVPool(kv, 2, 5*time.Second)
  717. defer pool.Close()
  718. schemas := NewSchemaManager(pool, "testdb")
  719. if err := schemas.CreateTable(&Schema{
  720. Name: "items",
  721. Columns: []Column{
  722. {Name: "id", Type: "INTEGER", PrimaryKey: true},
  723. {Name: "kind", Type: "TEXT"},
  724. },
  725. }); err != nil {
  726. t.Fatal(err)
  727. }
  728. if err := schemas.CreateIndex(&Index{
  729. Name: "idx_items_kind", Table: "items", Columns: []IndexColumn{{Name: "kind"}},
  730. }); err != nil {
  731. t.Fatal(err)
  732. }
  733. restarted := NewSchemaManager(pool, "testdb")
  734. getsBefore, _ := kv.readStats()
  735. if indexes, err := restarted.ListTableIndexes("items"); err != nil || len(indexes) != 1 {
  736. t.Fatalf("first list: indexes=%v err=%v", indexes, err)
  737. }
  738. getsAfterFirst, _ := kv.readStats()
  739. if getsAfterFirst <= getsBefore {
  740. t.Fatal("first index metadata lookup did not read durable metadata")
  741. }
  742. for i := 0; i < 10; i++ {
  743. if indexes, err := restarted.ListTableIndexes("items"); err != nil || len(indexes) != 1 {
  744. t.Fatalf("cached list %d: indexes=%v err=%v", i, indexes, err)
  745. }
  746. }
  747. getsAfterCached, _ := kv.readStats()
  748. if getsAfterCached != getsAfterFirst {
  749. t.Fatalf("cached index metadata issued %d extra reads", getsAfterCached-getsAfterFirst)
  750. }
  751. }
  752. func TestGetIndexDistinguishesNotFound(t *testing.T) {
  753. kv := newTestKVServer(t)
  754. defer kv.close()
  755. pool := newTestKVPool(kv, 4, 5*time.Second)
  756. defer pool.Close()
  757. schemas := NewSchemaManager(pool, "testdb")
  758. _, err := schemas.GetIndex("missing")
  759. if err == nil {
  760. t.Fatal("expected an error for a missing index")
  761. }
  762. if !errors.Is(err, ErrIndexNotFound) {
  763. t.Fatalf("expected ErrIndexNotFound, got %v", err)
  764. }
  765. }
  766. func TestListIndexesPropagatesCorruptJSON(t *testing.T) {
  767. kv := newTestKVServer(t)
  768. defer kv.close()
  769. pool := newTestKVPool(kv, 4, 5*time.Second)
  770. defer pool.Close()
  771. schemas := NewSchemaManager(pool, "testdb")
  772. if err := pool.WithClient(func(c *KVClient) error {
  773. return c.Write("testdb:indexes", "{not json")
  774. }); err != nil {
  775. t.Fatal(err)
  776. }
  777. if _, err := schemas.ListIndexes(); err == nil {
  778. t.Fatal("expected an error for corrupt index list JSON")
  779. }
  780. }
  781. func TestListTableIndexesSkipsOnlyNotFound(t *testing.T) {
  782. kv := newTestKVServer(t)
  783. defer kv.close()
  784. pool := newTestKVPool(kv, 4, 5*time.Second)
  785. defer pool.Close()
  786. schemas := NewSchemaManager(pool, "testdb")
  787. // Create a table and one index on it, then a dangling name in the list that
  788. // no longer has a definition (a concurrent drop), which must be skipped, not
  789. // surfaced.
  790. if err := schemas.CreateTable(&Schema{Name: "t", Columns: []Column{{Name: "id", Type: "INTEGER", PrimaryKey: true}, {Name: "v", Type: "TEXT"}}}); err != nil {
  791. t.Fatal(err)
  792. }
  793. if err := schemas.CreateIndex(&Index{Name: "uq_v", Table: "t", Unique: true, Columns: []IndexColumn{{Name: "v"}}}); err != nil {
  794. t.Fatal(err)
  795. }
  796. if err := schemas.addToIndexList("ghost_idx"); err != nil {
  797. t.Fatal(err)
  798. }
  799. indexes, err := schemas.ListTableIndexes("t")
  800. if err != nil {
  801. t.Fatalf("ListTableIndexes should skip the dangling index, got: %v", err)
  802. }
  803. if len(indexes) != 1 || indexes[0].Name != "uq_v" {
  804. t.Fatalf("expected only uq_v, got %v", indexes)
  805. }
  806. }