table_test.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451
  1. package storage
  2. import (
  3. "errors"
  4. "fmt"
  5. "testing"
  6. "time"
  7. )
  8. // TestSelectWithLimitStopsBeforeAllPages verifies that a limited scan stops
  9. // consuming pages as soon as the limit is satisfied instead of reading the
  10. // whole table.
  11. func TestSelectWithLimitStopsBeforeAllPages(t *testing.T) {
  12. kv := newTestKVServer(t)
  13. defer kv.close()
  14. pool := newTestKVPool(kv, 4, 5*time.Second)
  15. defer pool.Close()
  16. schemas := NewSchemaManager(pool, "testdb")
  17. tables := NewTableManager(pool, schemas, "testdb")
  18. if err := schemas.CreateTable(&Schema{
  19. Name: "t",
  20. Columns: []Column{
  21. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  22. {Name: "name", Type: "TEXT", Nullable: true},
  23. },
  24. }); err != nil {
  25. t.Fatalf("create table: %v", err)
  26. }
  27. const n = 20
  28. for i := int64(1); i <= n; i++ {
  29. if err := tables.Insert("t", Row{"id": i, "name": fmt.Sprintf("n%d", i)}); err != nil {
  30. t.Fatalf("insert %d: %v", i, err)
  31. }
  32. }
  33. // Force small pages so early termination is observable.
  34. kv.maxScanPage = 2
  35. _, nextsBefore, _ := kv.scanStats()
  36. rows, err := tables.SelectWithLimit("t", nil, 3, 0)
  37. if err != nil {
  38. t.Fatalf("SelectWithLimit: %v", err)
  39. }
  40. _, nextsAfter, _ := kv.scanStats()
  41. if len(rows) != 3 {
  42. t.Fatalf("SelectWithLimit returned %d rows, want 3", len(rows))
  43. }
  44. fullPages := (n + 1) / 2 // ceil(n / pageSize)
  45. if got := nextsAfter - nextsBefore; got >= fullPages {
  46. t.Fatalf("SelectWithLimit consumed %d pages, want < %d (should stop early)", got, fullPages)
  47. }
  48. }
  49. func TestPointUpdateDoesNotBlockPointReadBehindTableWriter(t *testing.T) {
  50. _, _, schemas, tables := newTestSession(t)
  51. createTestTable(t, schemas, "t", []Column{
  52. {Name: "id", Type: "INTEGER", PrimaryKey: true},
  53. {Name: "value", Type: "INTEGER"},
  54. })
  55. if err := tables.Insert("t", Row{"id": int64(1), "value": int64(1)}); err != nil {
  56. t.Fatal(err)
  57. }
  58. gate := tables.tableLock("t")
  59. gate.RLock()
  60. writerDone := make(chan error, 1)
  61. go func() {
  62. _, err := tables.Delete("t", func(Row) bool { return false })
  63. writerDone <- err
  64. }()
  65. deadline := time.Now().Add(time.Second)
  66. for gate.TryRLock() {
  67. gate.RUnlock()
  68. if time.Now().After(deadline) {
  69. gate.RUnlock()
  70. t.Fatal("table writer did not queue")
  71. }
  72. time.Sleep(time.Millisecond)
  73. }
  74. updateDone := make(chan error, 1)
  75. go func() {
  76. _, _, err := tables.UpdateByPK("t", "1", func(Row) (Row, error) {
  77. return Row{"value": int64(2)}, nil
  78. })
  79. updateDone <- err
  80. }()
  81. // Give the update time to reach the queued table gate. It must not hold the
  82. // row stripe while waiting, or this point read completes only after timeout.
  83. time.Sleep(10 * time.Millisecond)
  84. readDone := make(chan error, 1)
  85. go func() {
  86. _, err := tables.GetByPK("t", "1")
  87. readDone <- err
  88. }()
  89. select {
  90. case err := <-readDone:
  91. if err != nil {
  92. gate.RUnlock()
  93. t.Fatalf("point read: %v", err)
  94. }
  95. case <-time.After(250 * time.Millisecond):
  96. gate.RUnlock()
  97. <-writerDone
  98. <-updateDone
  99. <-readDone
  100. t.Fatal("point read deadlocked behind queued table writer")
  101. }
  102. gate.RUnlock()
  103. if err := <-writerDone; err != nil {
  104. t.Fatalf("table writer: %v", err)
  105. }
  106. if err := <-updateDone; err != nil {
  107. t.Fatalf("point update: %v", err)
  108. }
  109. }
  110. // TestDropTableDeletesDurableRows verifies that a direct DropTable removes all
  111. // durable row keys, not just the schema entry.
  112. func TestDropTableDeletesDurableRows(t *testing.T) {
  113. kv := newTestKVServer(t)
  114. defer kv.close()
  115. pool := newTestKVPool(kv, 4, 5*time.Second)
  116. defer pool.Close()
  117. schemas := NewSchemaManager(pool, "testdb")
  118. tables := NewTableManager(pool, schemas, "testdb")
  119. if err := schemas.CreateTable(&Schema{
  120. Name: "t",
  121. Columns: []Column{
  122. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  123. },
  124. }); err != nil {
  125. t.Fatalf("create table: %v", err)
  126. }
  127. for i := int64(1); i <= 5; i++ {
  128. if err := tables.Insert("t", Row{"id": i}); err != nil {
  129. t.Fatalf("insert %d: %v", i, err)
  130. }
  131. }
  132. if got := kv.countKeys("testdb:_data:t:"); got != 5 {
  133. t.Fatalf("expected 5 durable rows before drop, got %d", got)
  134. }
  135. if err := schemas.DropTable("t"); err != nil {
  136. t.Fatalf("DropTable: %v", err)
  137. }
  138. if got := kv.countKeys("testdb:_data:t:"); got != 0 {
  139. t.Fatalf("expected 0 durable rows after drop, got %d", got)
  140. }
  141. if kv.hasKey("testdb:_schema:t") {
  142. t.Fatalf("schema key still present after drop")
  143. }
  144. if kv.hasKey("testdb:_sys:rowid:t") {
  145. t.Fatalf("rowid counter key still present after drop")
  146. }
  147. }
  148. // TestScanKeysReturnsKeysOnly verifies that the key-only scan constructor
  149. // returns keys with empty values.
  150. func TestScanKeysReturnsKeysOnly(t *testing.T) {
  151. kv := newTestKVServer(t)
  152. defer kv.close()
  153. c := kv.client()
  154. defer c.Close()
  155. if _, err := c.Put([]byte("p:a"), []byte("value-a")); err != nil {
  156. t.Fatalf("put a: %v", err)
  157. }
  158. if _, err := c.Put([]byte("p:b"), []byte("value-b")); err != nil {
  159. t.Fatalf("put b: %v", err)
  160. }
  161. scan, err := c.ScanKeys([]byte("p:"))
  162. if err != nil {
  163. t.Fatalf("ScanKeys: %v", err)
  164. }
  165. defer scan.Close()
  166. entries, done, err := scan.Next()
  167. if err != nil {
  168. t.Fatalf("next: %v", err)
  169. }
  170. if !done || len(entries) != 2 {
  171. t.Fatalf("done=%v len=%d, want done and 2 entries", done, len(entries))
  172. }
  173. for _, e := range entries {
  174. if len(e.Key) == 0 {
  175. t.Fatalf("expected non-empty key")
  176. }
  177. if len(e.Value) != 0 {
  178. t.Fatalf("key-only scan returned a value %q for key %q", e.Value, e.Key)
  179. }
  180. }
  181. }
  182. // TestCountFastFirstDerivationUsesKeyOnlyScan verifies that the first-time
  183. // COUNT(*) derivation issues a key-only scan rather than pulling row values.
  184. func TestCountFastFirstDerivationUsesKeyOnlyScan(t *testing.T) {
  185. kv := newTestKVServer(t)
  186. defer kv.close()
  187. pool := newTestKVPool(kv, 4, 5*time.Second)
  188. defer pool.Close()
  189. schemas := NewSchemaManager(pool, "testdb")
  190. tables := NewTableManager(pool, schemas, "testdb")
  191. if err := schemas.CreateTable(&Schema{
  192. Name: "t",
  193. Columns: []Column{
  194. {Name: "id", Type: "INTEGER", Nullable: false, PrimaryKey: true},
  195. },
  196. }); err != nil {
  197. t.Fatalf("create table: %v", err)
  198. }
  199. for i := int64(1); i <= 5; i++ {
  200. if err := tables.Insert("t", Row{"id": i}); err != nil {
  201. t.Fatalf("insert %d: %v", i, err)
  202. }
  203. }
  204. before := kv.keyOnlyOpenCount()
  205. got, err := tables.CountFast("t")
  206. if err != nil {
  207. t.Fatalf("CountFast: %v", err)
  208. }
  209. after := kv.keyOnlyOpenCount()
  210. if got != 5 {
  211. t.Fatalf("CountFast = %d, want 5", got)
  212. }
  213. if after-before != 1 {
  214. t.Fatalf("expected CountFast first derivation to use one key-only scan, got %d", after-before)
  215. }
  216. }
  217. func TestInsertBulkStringPrimaryKeyCount(t *testing.T) {
  218. kv := newTestKVServer(t)
  219. defer kv.close()
  220. pool := newTestKVPool(kv, 4, 5*time.Second)
  221. defer pool.Close()
  222. schemas := NewSchemaManager(pool, "testdb")
  223. tables := NewTableManager(pool, schemas, "testdb")
  224. if err := schemas.CreateTable(&Schema{
  225. Name: "labels",
  226. Columns: []Column{
  227. {Name: "id", Type: "TEXT", Nullable: false, PrimaryKey: true},
  228. {Name: "value", Type: "BLOB", Nullable: true},
  229. },
  230. }); err != nil {
  231. t.Fatalf("create table: %v", err)
  232. }
  233. n, err := tables.InsertBulk("labels", []Row{
  234. {"id": "a", "value": []byte{0, 1, 2}},
  235. {"id": "b", "value": []byte{'|', '\r', '\n'}},
  236. })
  237. if err != nil {
  238. t.Fatalf("InsertBulk: %v", err)
  239. }
  240. if n != 2 {
  241. t.Fatalf("InsertBulk count = %d, want 2", n)
  242. }
  243. rows, err := tables.Select("labels", nil)
  244. if err != nil {
  245. t.Fatalf("Select: %v", err)
  246. }
  247. if len(rows) != 2 {
  248. t.Fatalf("Select returned %d rows, want 2", len(rows))
  249. }
  250. if n, err := tables.InsertBulk("labels", []Row{{"id": "a", "value": "duplicate"}}); err == nil || n != 0 {
  251. t.Fatalf("existing duplicate: n=%d err=%v", n, err)
  252. } else if got, want := err.Error(), "duplicate primary key: 'a'"; got != want {
  253. t.Fatalf("existing duplicate message = %q, want %q", got, want)
  254. }
  255. if n, err := tables.InsertBulk("labels", []Row{{"id": "c"}, {"id": "c"}}); err == nil || n != 0 {
  256. t.Fatalf("batch duplicate: n=%d err=%v", n, err)
  257. } else if got, want := err.Error(), "duplicate primary key: 'c'"; got != want {
  258. t.Fatalf("batch duplicate message = %q, want %q", got, want)
  259. }
  260. if _, err := tables.GetByPK("labels", "c"); err == nil {
  261. t.Fatal("duplicate batch persisted a row")
  262. }
  263. }
  264. // A single-column key reports its value; a TEXT key containing the row-key
  265. // separator or a quote is rendered as a SQL literal.
  266. func TestSingleColumnPrimaryKeyViolationMessage(t *testing.T) {
  267. _, _, schemas, tables := newTestSession(t)
  268. createTestTable(t, schemas, "nums", []Column{{Name: "id", Type: "INTEGER", PrimaryKey: true}})
  269. createTestTable(t, schemas, "texts", []Column{{Name: "id", Type: "TEXT", PrimaryKey: true}})
  270. cases := []struct {
  271. table string
  272. value interface{}
  273. want string
  274. }{
  275. {"nums", int64(1), "duplicate primary key: 1"},
  276. {"texts", "a:1", "duplicate primary key: 'a:1'"},
  277. {"texts", "o'brien", "duplicate primary key: 'o''brien'"},
  278. }
  279. for _, tc := range cases {
  280. if err := tables.Insert(tc.table, Row{"id": tc.value}); err != nil {
  281. t.Fatalf("insert %s %v: %v", tc.table, tc.value, err)
  282. }
  283. err := tables.Insert(tc.table, Row{"id": tc.value})
  284. var duplicate *PrimaryKeyViolationError
  285. if !errors.As(err, &duplicate) {
  286. t.Fatalf("duplicate insert %s %v error = %v, want PrimaryKeyViolationError", tc.table, tc.value, err)
  287. }
  288. if got := err.Error(); got != tc.want {
  289. t.Fatalf("duplicate insert %s %v message = %q, want %q", tc.table, tc.value, got, tc.want)
  290. }
  291. }
  292. }
  293. func TestSelectByIndexUsesPointReadsAfterBuild(t *testing.T) {
  294. kv := newTestKVServer(t)
  295. defer kv.close()
  296. pool := newTestKVPool(kv, 4, 5*time.Second)
  297. defer pool.Close()
  298. schemas := NewSchemaManager(pool, "testdb")
  299. tables := NewTableManager(pool, schemas, "testdb")
  300. if err := schemas.CreateTable(&Schema{
  301. Name: "items",
  302. Columns: []Column{
  303. {Name: "id", Type: "INTEGER", PrimaryKey: true},
  304. {Name: "kind", Type: "TEXT"},
  305. },
  306. }); err != nil {
  307. t.Fatal(err)
  308. }
  309. if err := schemas.CreateIndex(&Index{Name: "idx_kind", Table: "items", Columns: []IndexColumn{{Name: "kind"}}}); err != nil {
  310. t.Fatal(err)
  311. }
  312. for i := int64(1); i <= 20; i++ {
  313. if err := tables.Insert("items", Row{"id": i, "kind": fmt.Sprintf("k%d", i%2)}); err != nil {
  314. t.Fatal(err)
  315. }
  316. }
  317. if rows, err := tables.SelectByIndex("items", "idx_kind", "k1"); err != nil || len(rows) != 10 {
  318. t.Fatalf("initial indexed select: len=%d err=%v", len(rows), err)
  319. }
  320. opensBefore, _, _ := kv.scanStats()
  321. _, multiGetsBefore := kv.readStats()
  322. if rows, err := tables.SelectByIndex("items", "idx_kind", "k1"); err != nil || len(rows) != 10 {
  323. t.Fatalf("cached indexed select: len=%d err=%v", len(rows), err)
  324. }
  325. opensAfter, _, _ := kv.scanStats()
  326. _, multiGetsAfter := kv.readStats()
  327. if opensAfter != opensBefore {
  328. t.Fatalf("indexed select opened %d table scans after index build", opensAfter-opensBefore)
  329. }
  330. if multiGetsAfter-multiGetsBefore != 1 {
  331. t.Fatalf("indexed select issued %d multi-get requests, want 1", multiGetsAfter-multiGetsBefore)
  332. }
  333. }
  334. func TestPrimaryKeyMutationsDoNotScan(t *testing.T) {
  335. kv := newTestKVServer(t)
  336. defer kv.close()
  337. pool := newTestKVPool(kv, 4, 5*time.Second)
  338. defer pool.Close()
  339. schemas := NewSchemaManager(pool, "testdb")
  340. tables := NewTableManager(pool, schemas, "testdb")
  341. if err := schemas.CreateTable(&Schema{
  342. Name: "items",
  343. Columns: []Column{
  344. {Name: "id", Type: "TEXT", PrimaryKey: true},
  345. {Name: "value", Type: "INTEGER"},
  346. },
  347. }); err != nil {
  348. t.Fatal(err)
  349. }
  350. for i := 0; i < 20; i++ {
  351. if err := tables.Insert("items", Row{"id": fmt.Sprintf("item-%d", i), "value": int64(i)}); err != nil {
  352. t.Fatal(err)
  353. }
  354. }
  355. opensBefore, _, _ := kv.scanStats()
  356. oldRow, updated, err := tables.UpdateByPK("items", "item-10", func(Row) (Row, error) {
  357. return Row{"value": int64(99)}, nil
  358. })
  359. if err != nil || !updated || oldRow["value"] != int64(10) {
  360. t.Fatalf("point update: updated=%v old=%v err=%v", updated, oldRow, err)
  361. }
  362. deletedRow, deleted, err := tables.DeleteByPK("items", "item-11")
  363. if err != nil || !deleted || deletedRow["value"] != int64(11) {
  364. t.Fatalf("point delete: deleted=%v old=%v err=%v", deleted, deletedRow, err)
  365. }
  366. opensAfter, _, _ := kv.scanStats()
  367. if opensAfter != opensBefore {
  368. t.Fatalf("primary-key mutations opened %d scans", opensAfter-opensBefore)
  369. }
  370. row, err := tables.GetByPK("items", "item-10")
  371. if err != nil || row["value"] != int64(99) {
  372. t.Fatalf("updated row=%v err=%v", row, err)
  373. }
  374. if _, err := tables.GetByPK("items", "item-11"); err != ErrKeyNotFound {
  375. t.Fatalf("deleted row error=%v, want ErrKeyNotFound", err)
  376. }
  377. }
  378. func TestCountFastResetsAfterDirectDropAndRecreate(t *testing.T) {
  379. kv := newTestKVServer(t)
  380. defer kv.close()
  381. pool := newTestKVPool(kv, 2, 5*time.Second)
  382. defer pool.Close()
  383. schemas := NewSchemaManager(pool, "testdb")
  384. tables := NewTableManager(pool, schemas, "testdb")
  385. create := func() {
  386. if err := schemas.CreateTable(&Schema{Name: "events", Columns: []Column{{Name: "id", Type: "INTEGER", PrimaryKey: true}}}); err != nil {
  387. t.Fatal(err)
  388. }
  389. }
  390. create()
  391. if err := tables.Insert("events", Row{"id": int64(1)}); err != nil {
  392. t.Fatal(err)
  393. }
  394. if count, err := tables.CountFast("events"); err != nil || count != 1 {
  395. t.Fatalf("initial count=%d err=%v", count, err)
  396. }
  397. if err := schemas.DropTable("events"); err != nil {
  398. t.Fatal(err)
  399. }
  400. create()
  401. if count, err := tables.CountFast("events"); err != nil || count != 0 {
  402. t.Fatalf("recreated count=%d err=%v", count, err)
  403. }
  404. }