Skip to content

Commit 070c578

Browse files
zeripathtechknowlogick6543
authored
Fix postgres ID sequences broken by recreate-table (#15015)
* Fix postgres ID sequences broken by recreate-table Unfortunately there is a subtle problem with recreatetable on postgres which leads to the sequences not being renamed and not being left at 0. Fix #14725 Signed-off-by: Andrew Thornton <[email protected]> * let us try information_schema instead Signed-off-by: Andrew Thornton <[email protected]> * try again Signed-off-by: Andrew Thornton <[email protected]> Co-authored-by: techknowlogick <[email protected]> Co-authored-by: 6543 <[email protected]>
1 parent 71aca93 commit 070c578

File tree

4 files changed

+198
-1
lines changed

4 files changed

+198
-1
lines changed

‎models/consistency.go

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@
55
package models
66

77
import (
8+
"fmt"
89
"reflect"
10+
"regexp"
911
"strings"
1012
"testing"
1113

14+
"code.gitea.io/gitea/modules/setting"
1215
"github.com/stretchr/testify/assert"
1316
"xorm.io/builder"
1417
)
@@ -315,3 +318,61 @@ func CountCommentTypeLabelWithEmptyLabel() (int64, error) {
315318
func FixCommentTypeLabelWithEmptyLabel() (int64, error) {
316319
return x.Where(builder.Eq{"type": CommentTypeLabel, "label_id": 0}).Delete(new(Comment))
317320
}
321+
322+
// CountBadSequences looks for broken sequences from recreate-table mistakes
323+
func CountBadSequences() (int64, error) {
324+
if !setting.Database.UsePostgreSQL {
325+
return 0, nil
326+
}
327+
328+
sess := x.NewSession()
329+
defer sess.Close()
330+
331+
var sequences []string
332+
schema := sess.Engine().Dialect().URI().Schema
333+
334+
sess.Engine().SetSchema("")
335+
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
336+
return 0, err
337+
}
338+
sess.Engine().SetSchema(schema)
339+
340+
return int64(len(sequences)), nil
341+
}
342+
343+
// FixBadSequences fixes for broken sequences from recreate-table mistakes
344+
func FixBadSequences() error {
345+
if !setting.Database.UsePostgreSQL {
346+
return nil
347+
}
348+
349+
sess := x.NewSession()
350+
defer sess.Close()
351+
if err := sess.Begin(); err != nil {
352+
return err
353+
}
354+
355+
var sequences []string
356+
schema := sess.Engine().Dialect().URI().Schema
357+
358+
sess.Engine().SetSchema("")
359+
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
360+
return err
361+
}
362+
sess.Engine().SetSchema(schema)
363+
364+
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
365+
366+
for _, sequence := range sequences {
367+
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
368+
newSequenceName := tableName + "_id_seq"
369+
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
370+
return err
371+
}
372+
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
373+
return err
374+
}
375+
}
376+
377+
return sess.Commit()
378+
}

‎models/migrations/migrations.go

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,8 @@ var migrations = []Migration{
296296
NewMigration("Add time_id column to Comment", addTimeIDCommentColumn),
297297
// v174 -> v175
298298
NewMigration("create repo transfer table", addRepoTransfer),
299+
// v175 -> v176
300+
NewMigration("Fix Postgres ID Sequences broken by recreate-table", fixPostgresIDSequences),
299301
}
300302

301303
// GetCurrentDBVersion returns the current db version
@@ -565,6 +567,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
565567
return err
566568
}
567569
case setting.Database.UsePostgreSQL:
570+
var originalSequences []string
571+
type sequenceData struct {
572+
LastValue int `xorm:"'last_value'"`
573+
IsCalled bool `xorm:"'is_called'"`
574+
}
575+
sequenceMap := map[string]sequenceData{}
576+
577+
schema := sess.Engine().Dialect().URI().Schema
578+
sess.Engine().SetSchema("")
579+
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
580+
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
581+
return err
582+
}
583+
sess.Engine().SetSchema(schema)
584+
585+
for _, sequence := range originalSequences {
586+
sequenceData := sequenceData{}
587+
if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
588+
log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
589+
return err
590+
}
591+
sequenceMap[sequence] = sequenceData
592+
593+
}
594+
568595
// CASCADE causes postgres to drop all the constraints on the old table
569596
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
570597
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@@ -578,7 +605,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
578605
}
579606

580607
var indices []string
581-
schema := sess.Engine().Dialect().URI().Schema
582608
sess.Engine().SetSchema("")
583609
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
584610
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@@ -594,6 +620,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
594620
}
595621
}
596622

623+
var sequences []string
624+
sess.Engine().SetSchema("")
625+
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
626+
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
627+
return err
628+
}
629+
sess.Engine().SetSchema(schema)
630+
631+
for _, sequence := range sequences {
632+
newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
633+
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
634+
log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
635+
return err
636+
}
637+
val, ok := sequenceMap[newSequenceName]
638+
if newSequenceName == tableName+"_id_seq" {
639+
if ok && val.LastValue != 0 {
640+
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
641+
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
642+
return err
643+
}
644+
} else {
645+
// We're going to try to guess this
646+
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
647+
log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
648+
return err
649+
}
650+
}
651+
} else if ok {
652+
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
653+
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
654+
return err
655+
}
656+
}
657+
658+
}
659+
597660
case setting.Database.UseMSSQL:
598661
// MSSQL will drop all the constraints on the old table
599662
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {

‎models/migrations/v175.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2021 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package migrations
6+
7+
import (
8+
"fmt"
9+
"regexp"
10+
11+
"code.gitea.io/gitea/modules/log"
12+
"code.gitea.io/gitea/modules/setting"
13+
"xorm.io/xorm"
14+
)
15+
16+
func fixPostgresIDSequences(x *xorm.Engine) error {
17+
if !setting.Database.UsePostgreSQL {
18+
return nil
19+
}
20+
21+
sess := x.NewSession()
22+
defer sess.Close()
23+
if err := sess.Begin(); err != nil {
24+
return err
25+
}
26+
27+
var sequences []string
28+
schema := sess.Engine().Dialect().URI().Schema
29+
30+
sess.Engine().SetSchema("")
31+
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
32+
log.Error("Unable to find sequences: %v", err)
33+
return err
34+
}
35+
sess.Engine().SetSchema(schema)
36+
37+
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
38+
39+
for _, sequence := range sequences {
40+
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
41+
newSequenceName := tableName + "_id_seq"
42+
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
43+
log.Error("Unable to rename %s to %s. Error: %v", sequence, newSequenceName, err)
44+
return err
45+
}
46+
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
47+
log.Error("Unable to reset sequence %s for %s. Error: %v", newSequenceName, tableName, err)
48+
return err
49+
}
50+
}
51+
52+
return sess.Commit()
53+
}

‎modules/doctor/dbconsistency.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"code.gitea.io/gitea/models"
1111
"code.gitea.io/gitea/models/migrations"
1212
"code.gitea.io/gitea/modules/log"
13+
"code.gitea.io/gitea/modules/setting"
1314
)
1415

1516
func checkDBConsistency(logger log.Logger, autofix bool) error {
@@ -131,6 +132,25 @@ func checkDBConsistency(logger log.Logger, autofix bool) error {
131132
}
132133
// TODO: function to recalc all counters
133134

135+
if setting.Database.UsePostgreSQL {
136+
count, err = models.CountBadSequences()
137+
if err != nil {
138+
logger.Critical("Error: %v whilst checking sequence values")
139+
}
140+
if count > 0 {
141+
if autofix {
142+
err := models.FixBadSequences()
143+
if err != nil {
144+
logger.Critical("Error: %v whilst attempting to fix sequences")
145+
return err
146+
}
147+
logger.Info("%d sequences updated", count)
148+
} else {
149+
logger.Warn("%d sequences with incorrect values", count)
150+
}
151+
}
152+
}
153+
134154
return nil
135155
}
136156

0 commit comments

Comments
 (0)