row_locks_test.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. package pgserver
  2. import (
  3. "context"
  4. "fmt"
  5. "net"
  6. "strings"
  7. "sync"
  8. "testing"
  9. "time"
  10. "github.com/jackc/pgx/v5"
  11. "github.com/danfragoso/pizzasql-next/pkg/storage"
  12. "github.com/danfragoso/pizzasql-next/pkg/testkv"
  13. )
  14. // connectPGXPair starts one wire server and returns two independent client
  15. // connections to the same database so row-lock contention can be exercised.
  16. func connectPGXPair(t *testing.T) (*pgx.Conn, *pgx.Conn) {
  17. t.Helper()
  18. conns := connectPGXMany(t, 2)
  19. return conns[0], conns[1]
  20. }
  21. func connectPGXMany(t *testing.T, n int) []*pgx.Conn {
  22. t.Helper()
  23. kv := testkv.New(t)
  24. pool := kv.Pool(8)
  25. t.Cleanup(func() { pool.Close() })
  26. dbManager := storage.NewDatabaseManager(pool, &storage.DatabaseManagerConfig{
  27. DefaultDatabase: "rowlocks",
  28. AutoCreate: true,
  29. })
  30. listener, err := net.Listen("tcp", "127.0.0.1:0")
  31. if err != nil {
  32. t.Fatal(err)
  33. }
  34. t.Cleanup(func() { listener.Close() })
  35. var wg sync.WaitGroup
  36. t.Cleanup(wg.Wait)
  37. go func() {
  38. for {
  39. server, err := listener.Accept()
  40. if err != nil {
  41. return
  42. }
  43. wg.Add(1)
  44. go func() {
  45. defer wg.Done()
  46. _ = NewConnection(server, dbManager, true).Handle()
  47. }()
  48. }
  49. }()
  50. dsn := "postgres://tester@" + listener.Addr().String() + "/rowlocks?sslmode=disable"
  51. connect := func() *pgx.Conn {
  52. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  53. defer cancel()
  54. conn, err := pgx.Connect(ctx, dsn)
  55. if err != nil {
  56. t.Fatalf("pgx connect: %v", err)
  57. }
  58. t.Cleanup(func() { _ = conn.Close(context.Background()) })
  59. return conn
  60. }
  61. conns := make([]*pgx.Conn, 0, n)
  62. for i := 0; i < n; i++ {
  63. conns = append(conns, connect())
  64. }
  65. return conns
  66. }
  67. func TestRowLockSkipLockedClaimsOneRowPerTransaction(t *testing.T) {
  68. a, b := connectPGXPair(t)
  69. ctx := context.Background()
  70. for conn, statements := range map[*pgx.Conn][]string{
  71. a: {
  72. `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT, owner TEXT)`,
  73. `INSERT INTO jobs VALUES (1, 'pending', NULL), (2, 'pending', NULL), (3, 'pending', NULL)`,
  74. },
  75. } {
  76. for _, sql := range statements {
  77. if _, err := conn.Exec(ctx, sql); err != nil {
  78. t.Fatalf("%s: %v", sql, err)
  79. }
  80. }
  81. }
  82. claim := func(conn *pgx.Conn, owner string) int32 {
  83. tx, err := conn.Begin(ctx)
  84. if err != nil {
  85. t.Fatal(err)
  86. }
  87. defer tx.Rollback(ctx)
  88. var id int32
  89. if err := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE status = 'pending' ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id); err != nil {
  90. t.Fatalf("%s claim: %v", owner, err)
  91. }
  92. if _, err := tx.Exec(ctx, `UPDATE jobs SET status = 'taken', owner = $1 WHERE id = $2`, owner, id); err != nil {
  93. t.Fatalf("%s update: %v", owner, err)
  94. }
  95. if err := tx.Commit(ctx); err != nil {
  96. t.Fatalf("%s commit: %v", owner, err)
  97. }
  98. return id
  99. }
  100. // The first transaction keeps its lock open while the second claims.
  101. tx, err := a.Begin(ctx)
  102. if err != nil {
  103. t.Fatal(err)
  104. }
  105. defer tx.Rollback(ctx)
  106. var first int32
  107. if err := tx.QueryRow(ctx, `SELECT id FROM jobs WHERE status = 'pending' ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&first); err != nil {
  108. t.Fatalf("first claim: %v", err)
  109. }
  110. if first != 1 {
  111. t.Fatalf("first claim = %d, want 1", first)
  112. }
  113. second := claim(b, "b")
  114. if second == first {
  115. t.Fatalf("both transactions claimed row %d", first)
  116. }
  117. if _, err := tx.Exec(ctx, `UPDATE jobs SET status = 'taken', owner = 'a' WHERE id = $1`, first); err != nil {
  118. t.Fatalf("first update: %v", err)
  119. }
  120. if err := tx.Commit(ctx); err != nil {
  121. t.Fatalf("first commit: %v", err)
  122. }
  123. rows, err := a.Query(ctx, `SELECT id, status, owner FROM jobs ORDER BY id`)
  124. if err != nil {
  125. t.Fatal(err)
  126. }
  127. defer rows.Close()
  128. count := 0
  129. taken := 0
  130. for rows.Next() {
  131. var id int32
  132. var status string
  133. var owner *string
  134. if err := rows.Scan(&id, &status, &owner); err != nil {
  135. t.Fatal(err)
  136. }
  137. if status == "taken" {
  138. if owner == nil {
  139. t.Fatalf("row %d is taken without an owner", id)
  140. }
  141. taken++
  142. } else if id != 3 || status != "pending" {
  143. t.Fatalf("row %d = %s/%v, want pending or taken", id, status, owner)
  144. }
  145. count++
  146. }
  147. if count != 3 || taken != 2 {
  148. t.Fatalf("rows/taken = %d/%d, want 3/2", count, taken)
  149. }
  150. }
  151. func TestRowLockNowaitAndWriterConflict(t *testing.T) {
  152. a, b := connectPGXPair(t)
  153. ctx := context.Background()
  154. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  155. t.Fatal(err)
  156. }
  157. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending'), (2, 'pending')`); err != nil {
  158. t.Fatal(err)
  159. }
  160. tx, err := a.Begin(ctx)
  161. if err != nil {
  162. t.Fatal(err)
  163. }
  164. defer tx.Rollback(ctx)
  165. if _, err := tx.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  166. t.Fatalf("lock: %v", err)
  167. }
  168. _, err = b.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE NOWAIT`)
  169. if err == nil || !strings.Contains(err.Error(), "55P03") {
  170. t.Fatalf("NOWAIT error = %v, want SQLSTATE 55P03", err)
  171. }
  172. // A writer that does not use a locking clause must not commit over the
  173. // lock; it gets the optimistic serialization failure.
  174. _, err = b.Exec(ctx, `UPDATE jobs SET status = 'stolen' WHERE id = 1`)
  175. if err == nil || !strings.Contains(err.Error(), "40001") {
  176. t.Fatalf("writer error = %v, want SQLSTATE 40001", err)
  177. }
  178. }
  179. func TestRowLockBlockingWaitsForCommit(t *testing.T) {
  180. a, b := connectPGXPair(t)
  181. ctx := context.Background()
  182. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  183. t.Fatal(err)
  184. }
  185. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending')`); err != nil {
  186. t.Fatal(err)
  187. }
  188. tx, err := a.Begin(ctx)
  189. if err != nil {
  190. t.Fatal(err)
  191. }
  192. if _, err := tx.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  193. t.Fatal(err)
  194. }
  195. done := make(chan error, 1)
  196. go func() {
  197. var id int32
  198. err := b.QueryRow(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`).Scan(&id)
  199. done <- err
  200. }()
  201. select {
  202. case err := <-done:
  203. t.Fatalf("blocking FOR UPDATE returned before commit: %v", err)
  204. case <-time.After(150 * time.Millisecond):
  205. }
  206. if err := tx.Commit(ctx); err != nil {
  207. t.Fatal(err)
  208. }
  209. select {
  210. case err := <-done:
  211. if err != nil {
  212. t.Fatalf("blocking FOR UPDATE: %v", err)
  213. }
  214. case <-time.After(5 * time.Second):
  215. t.Fatal("blocking FOR UPDATE did not resume")
  216. }
  217. }
  218. func TestRowLockDeadlockDetected(t *testing.T) {
  219. a, b := connectPGXPair(t)
  220. ctx := context.Background()
  221. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY)`); err != nil {
  222. t.Fatal(err)
  223. }
  224. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1), (2)`); err != nil {
  225. t.Fatal(err)
  226. }
  227. ta, err := a.Begin(ctx)
  228. if err != nil {
  229. t.Fatal(err)
  230. }
  231. defer ta.Rollback(ctx)
  232. tb, err := b.Begin(ctx)
  233. if err != nil {
  234. t.Fatal(err)
  235. }
  236. defer tb.Rollback(ctx)
  237. if _, err := ta.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  238. t.Fatal(err)
  239. }
  240. if _, err := tb.Exec(ctx, `SELECT id FROM jobs WHERE id = 2 FOR UPDATE`); err != nil {
  241. t.Fatal(err)
  242. }
  243. blocked := make(chan error, 1)
  244. go func() {
  245. _, err := ta.Exec(ctx, `SELECT id FROM jobs WHERE id = 2 FOR UPDATE`)
  246. blocked <- err
  247. }()
  248. time.Sleep(100 * time.Millisecond)
  249. _, err = tb.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`)
  250. if err == nil || !strings.Contains(err.Error(), "40P01") {
  251. t.Fatalf("deadlock error = %v, want SQLSTATE 40P01", err)
  252. }
  253. if err := tb.Rollback(ctx); err != nil {
  254. t.Fatal(err)
  255. }
  256. select {
  257. case err := <-blocked:
  258. if err != nil {
  259. t.Fatalf("waiting transaction after deadlock peer rollback: %v", err)
  260. }
  261. case <-time.After(5 * time.Second):
  262. t.Fatal("waiting transaction did not resume")
  263. }
  264. }
  265. func TestRowLockShareAllowsReadersAndBlocksWriter(t *testing.T) {
  266. a, b := connectPGXPair(t)
  267. ctx := context.Background()
  268. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  269. t.Fatal(err)
  270. }
  271. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending')`); err != nil {
  272. t.Fatal(err)
  273. }
  274. ta, err := a.Begin(ctx)
  275. if err != nil {
  276. t.Fatal(err)
  277. }
  278. defer ta.Rollback(ctx)
  279. tb, err := b.Begin(ctx)
  280. if err != nil {
  281. t.Fatal(err)
  282. }
  283. defer tb.Rollback(ctx)
  284. if _, err := ta.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR SHARE`); err != nil {
  285. t.Fatal(err)
  286. }
  287. if _, err := tb.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR SHARE`); err != nil {
  288. t.Fatalf("second FOR SHARE: %v", err)
  289. }
  290. if _, err := tb.Exec(ctx, `UPDATE jobs SET status = 'taken' WHERE id = 1`); err != nil {
  291. t.Fatalf("staging writer: %v", err)
  292. }
  293. err = tb.Commit(ctx)
  294. if err == nil || !strings.Contains(err.Error(), "40001") {
  295. t.Fatalf("writer with share lock held error = %v, want 40001", err)
  296. }
  297. }
  298. func TestRowLockAutocommitReleasesImmediately(t *testing.T) {
  299. a, b := connectPGXPair(t)
  300. ctx := context.Background()
  301. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  302. t.Fatal(err)
  303. }
  304. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending')`); err != nil {
  305. t.Fatal(err)
  306. }
  307. if _, err := a.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  308. t.Fatal(err)
  309. }
  310. if _, err := b.Exec(ctx, `UPDATE jobs SET status = 'taken' WHERE id = 1`); err != nil {
  311. t.Fatalf("autocommit lock was not released: %v", err)
  312. }
  313. }
  314. func TestRowLockQueueWorkersDoNotDoubleClaim(t *testing.T) {
  315. const workers = 4
  316. const jobs = 40
  317. conns := connectPGXMany(t, workers)
  318. ctx := context.Background()
  319. if _, err := conns[0].Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT, owner TEXT)`); err != nil {
  320. t.Fatal(err)
  321. }
  322. var insert strings.Builder
  323. insert.WriteString(`INSERT INTO jobs VALUES `)
  324. for i := 1; i <= jobs; i++ {
  325. if i > 1 {
  326. insert.WriteString(", ")
  327. }
  328. fmt.Fprintf(&insert, "(%d, 'pending', NULL)", i)
  329. }
  330. if _, err := conns[0].Exec(ctx, insert.String()); err != nil {
  331. t.Fatal(err)
  332. }
  333. var wg sync.WaitGroup
  334. errs := make(chan error, workers)
  335. for w, conn := range conns {
  336. wg.Add(1)
  337. go func(w int, conn *pgx.Conn) {
  338. defer wg.Done()
  339. owner := fmt.Sprintf("worker-%d", w)
  340. for {
  341. tx, err := conn.Begin(ctx)
  342. if err != nil {
  343. errs <- err
  344. return
  345. }
  346. var id int32
  347. err = tx.QueryRow(ctx, `SELECT id FROM jobs WHERE status = 'pending' ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&id)
  348. if err != nil {
  349. _ = tx.Rollback(ctx)
  350. if err == pgx.ErrNoRows {
  351. return
  352. }
  353. errs <- err
  354. return
  355. }
  356. if _, err := tx.Exec(ctx, `UPDATE jobs SET status = 'taken', owner = $1 WHERE id = $2`, owner, id); err != nil {
  357. _ = tx.Rollback(ctx)
  358. errs <- err
  359. return
  360. }
  361. if err := tx.Commit(ctx); err != nil {
  362. errs <- err
  363. return
  364. }
  365. }
  366. }(w, conn)
  367. }
  368. wg.Wait()
  369. close(errs)
  370. for err := range errs {
  371. t.Fatalf("worker error: %v", err)
  372. }
  373. var taken, pending int
  374. if err := conns[0].QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE status = 'taken'`).Scan(&taken); err != nil {
  375. t.Fatal(err)
  376. }
  377. if err := conns[0].QueryRow(ctx, `SELECT COUNT(*) FROM jobs WHERE status = 'pending'`).Scan(&pending); err != nil {
  378. t.Fatal(err)
  379. }
  380. if taken != jobs || pending != 0 {
  381. t.Fatalf("taken/pending = %d/%d, want %d/0", taken, pending, jobs)
  382. }
  383. var distinctOwners int
  384. if err := conns[0].QueryRow(ctx, `SELECT COUNT(DISTINCT id) FROM jobs`).Scan(&distinctOwners); err != nil {
  385. t.Fatal(err)
  386. }
  387. if distinctOwners != jobs {
  388. t.Fatalf("distinct ids = %d, want %d", distinctOwners, jobs)
  389. }
  390. }
  391. func TestRowLockAliasedTableAndEmptySkip(t *testing.T) {
  392. a, b := connectPGXPair(t)
  393. ctx := context.Background()
  394. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  395. t.Fatal(err)
  396. }
  397. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending'), (2, 'pending')`); err != nil {
  398. t.Fatal(err)
  399. }
  400. tx, err := a.Begin(ctx)
  401. if err != nil {
  402. t.Fatal(err)
  403. }
  404. defer tx.Rollback(ctx)
  405. var id int32
  406. if err := tx.QueryRow(ctx, `SELECT j.id FROM jobs AS j WHERE j.id = 1 FOR UPDATE`).Scan(&id); err != nil {
  407. t.Fatalf("aliased locking select: %v", err)
  408. }
  409. if id != 1 {
  410. t.Fatalf("aliased id = %d", id)
  411. }
  412. if err := tx.Commit(ctx); err != nil {
  413. t.Fatal(err)
  414. }
  415. // Hold both rows, then confirm SKIP LOCKED reports no candidate.
  416. ta, err := a.Begin(ctx)
  417. if err != nil {
  418. t.Fatal(err)
  419. }
  420. defer ta.Rollback(ctx)
  421. if _, err := ta.Exec(ctx, `SELECT id FROM jobs FOR UPDATE`); err != nil {
  422. t.Fatal(err)
  423. }
  424. tb, err := b.Begin(ctx)
  425. if err != nil {
  426. t.Fatal(err)
  427. }
  428. defer tb.Rollback(ctx)
  429. var n int32
  430. if err := tb.QueryRow(ctx, `SELECT id FROM jobs WHERE status = 'pending' LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&n); err != pgx.ErrNoRows {
  431. t.Fatalf("all-locked SKIP LOCKED = %v, want ErrNoRows", err)
  432. }
  433. }
  434. func TestRowLockRollbackReleases(t *testing.T) {
  435. a, b := connectPGXPair(t)
  436. ctx := context.Background()
  437. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  438. t.Fatal(err)
  439. }
  440. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending')`); err != nil {
  441. t.Fatal(err)
  442. }
  443. tx, err := a.Begin(ctx)
  444. if err != nil {
  445. t.Fatal(err)
  446. }
  447. if _, err := tx.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  448. t.Fatal(err)
  449. }
  450. if err := tx.Rollback(ctx); err != nil {
  451. t.Fatal(err)
  452. }
  453. if _, err := b.Exec(ctx, `UPDATE jobs SET status = 'taken' WHERE id = 1`); err != nil {
  454. t.Fatalf("rollback did not release lock: %v", err)
  455. }
  456. }
  457. func TestRowLockBlocksAutocommitDDL(t *testing.T) {
  458. a, b := connectPGXPair(t)
  459. ctx := context.Background()
  460. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  461. t.Fatal(err)
  462. }
  463. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending')`); err != nil {
  464. t.Fatal(err)
  465. }
  466. tx, err := a.Begin(ctx)
  467. if err != nil {
  468. t.Fatal(err)
  469. }
  470. defer tx.Rollback(ctx)
  471. if _, err := tx.Exec(ctx, `SELECT id FROM jobs FOR UPDATE`); err != nil {
  472. t.Fatal(err)
  473. }
  474. if _, err := b.Exec(ctx, `ALTER TABLE jobs ADD COLUMN note TEXT`); err == nil || !strings.Contains(err.Error(), "40001") {
  475. t.Fatalf("DDL with row lock held error = %v, want 40001", err)
  476. }
  477. }
  478. func TestRowLockSyntheticRowIDTable(t *testing.T) {
  479. a, _ := connectPGXPair(t)
  480. ctx := context.Background()
  481. if _, err := a.Exec(ctx, `CREATE TABLE notes (body TEXT)`); err != nil {
  482. t.Fatal(err)
  483. }
  484. if _, err := a.Exec(ctx, `INSERT INTO notes VALUES ('hello')`); err != nil {
  485. t.Fatal(err)
  486. }
  487. var body string
  488. if err := a.QueryRow(ctx, `SELECT body FROM notes FOR UPDATE`).Scan(&body); err != nil {
  489. t.Fatalf("locking synthetic-rowid table: %v", err)
  490. }
  491. if body != "hello" {
  492. t.Fatalf("body = %q", body)
  493. }
  494. }
  495. func TestRowLockSkippedPrimaryKeyDoesNotPoisonCommit(t *testing.T) {
  496. a, b := connectPGXPair(t)
  497. ctx := context.Background()
  498. if _, err := a.Exec(ctx, `CREATE TABLE jobs (id INTEGER PRIMARY KEY, status TEXT)`); err != nil {
  499. t.Fatal(err)
  500. }
  501. if _, err := a.Exec(ctx, `INSERT INTO jobs VALUES (1, 'pending'), (2, 'pending')`); err != nil {
  502. t.Fatal(err)
  503. }
  504. ta, err := a.Begin(ctx)
  505. if err != nil {
  506. t.Fatal(err)
  507. }
  508. if _, err := ta.Exec(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE`); err != nil {
  509. t.Fatal(err)
  510. }
  511. tb, err := b.Begin(ctx)
  512. if err != nil {
  513. t.Fatal(err)
  514. }
  515. var id int32
  516. if err := tb.QueryRow(ctx, `SELECT id FROM jobs WHERE id = 1 FOR UPDATE SKIP LOCKED`).Scan(&id); err != pgx.ErrNoRows {
  517. t.Fatalf("skipped primary key = %v, want ErrNoRows", err)
  518. }
  519. if _, err := tb.Exec(ctx, `UPDATE jobs SET status = 'b' WHERE id = 2`); err != nil {
  520. t.Fatal(err)
  521. }
  522. if _, err := ta.Exec(ctx, `UPDATE jobs SET status = 'a' WHERE id = 1`); err != nil {
  523. t.Fatal(err)
  524. }
  525. if err := ta.Commit(ctx); err != nil {
  526. t.Fatal(err)
  527. }
  528. if err := tb.Commit(ctx); err != nil {
  529. t.Fatalf("skipped read poisoned commit: %v", err)
  530. }
  531. }