Skip to content

Commit 5bf82cd

Browse files
geroplgeropl
authored
Move towards sync deletion (away from PeriodicDeleter) - step II/II (#18866)
* [db] DBTeam: drop deleted column (unused) * [db] DBProject: drop deleted column (unused) * [db] Drop table d_b_user_storage_resource * [db] DBTokenEntry: drop deleted column (unused) * [db] DBAuthProviderEntry: drop deleted column (unused) * [db] DBGitpodToken: drop deleted column (unused) * [db] DBTeamMembership: drop deleted column (unused) * [db] DBProjectUsage: drop deleted column (unused) * [db] DBUserSshPublicKey: drop deleted column (unused) * [server] Fix flaky test * [db] Make backwards-compatible to mysql 5.7
1 parent 0f926eb commit 5bf82cd

29 files changed

+262
-149
lines changed

‎components/gitpod-db/go/organization.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,6 @@ type Organization struct {
2323
LastModified time.Time `gorm:"column:_lastModified;type:timestamp;default:CURRENT_TIMESTAMP(6);" json:"_lastModified"`
2424

2525
MarkedDeleted bool `gorm:"column:markedDeleted;type:tinyint;default:0;" json:"marked_deleted"`
26-
27-
// deleted is reserved for use by periodic deleter
28-
_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
2926
}
3027

3128
// TableName sets the insert table name for this struct type
@@ -85,7 +82,6 @@ func GetSingleOrganizationWithActiveSSO(ctx context.Context, conn *gorm.DB) (Org
8582
WithContext(ctx).
8683
Table(fmt.Sprintf("%s as team", (&Organization{}).TableName())).
8784
Joins(fmt.Sprintf("JOIN %s AS config ON team.id = config.organizationId", (&OIDCClientConfig{}).TableName())).
88-
Where("team.deleted = ?", 0).
8985
Where("config.deleted = ?", 0).
9086
Where("config.active = ?", 1).
9187
Find(&orgs)

‎components/gitpod-db/go/organization_membership.go

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ type OrganizationMembership struct {
2424
CreationTime VarcharTime `gorm:"column:creationTime;type:varchar;size:255;" json:"creationTime"`
2525
// Read-only (-> property).
2626
LastModified time.Time `gorm:"->:column:_lastModified;type:timestamp;default:CURRENT_TIMESTAMP(6);" json:"_lastModified"`
27-
28-
// deleted column is reserved for use by periodic deleter
29-
_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
3027
}
3128

3229
// TableName sets the insert table name for this struct type
@@ -54,7 +51,6 @@ func GetOrganizationMembership(ctx context.Context, conn *gorm.DB, userID, orgID
5451
tx := conn.WithContext(ctx).
5552
Where("userId = ?", userID.String()).
5653
Where("teamId = ?", orgID.String()).
57-
Where("deleted = ?", false).
5854
First(&membership)
5955
if tx.Error != nil {
6056
if errors.Is(tx.Error, gorm.ErrRecordNotFound) {
@@ -79,8 +75,7 @@ func DeleteOrganizationMembership(ctx context.Context, conn *gorm.DB, userID uui
7975
Model(&OrganizationMembership{}).
8076
Where("userId = ?", userID.String()).
8177
Where("teamId = ?", orgID.String()).
82-
Where("deleted = ?", 0).
83-
Update("deleted", 1)
78+
Delete(&OrganizationMembership{})
8479
if tx.Error != nil {
8580
return fmt.Errorf("failed to retrieve organization membership for user %s, organization %s: %w", userID.String(), orgID.String(), tx.Error)
8681
}

‎components/gitpod-db/go/project.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@ type Project struct {
2828
UserID sql.NullString `gorm:"column:userId;type:char;size:36;" json:"userId"`
2929

3030
MarkedDeleted bool `gorm:"column:markedDeleted;type:tinyint;default:0;" json:"markedDeleted"`
31-
32-
// deleted is reserved for use by periodic deleter
33-
_ bool `gorm:"column:deleted;type:tinyint;default:0;" json:"deleted"`
3431
}
3532

3633
// TableName sets the insert table name for this struct type

‎components/gitpod-db/go/project_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ var projectJSON = map[string]interface{}{
2323
"teamId": "0e433063-1358-4892-9ed2-68e273d17d07",
2424
"appInstallationId": "20446411",
2525
"creationTime": "2021-11-01T19:36:07.532Z",
26-
"deleted": 0,
2726
"_lastModified": "2021-11-02 10:49:12.473658",
2827
"name": "gptest1-repo1-private",
2928
"markedDeleted": 1,
@@ -49,7 +48,7 @@ func TestProject_ReadExistingRecords(t *testing.T) {
4948

5049
func insertRawProject(t *testing.T, conn *gorm.DB, obj map[string]interface{}) uuid.UUID {
5150
columns := []string{
52-
"id", "cloneUrl", "teamId", "appInstallationId", "creationTime", "deleted", "_lastModified", "name", "markedDeleted", "userId", "slug", "settings",
51+
"id", "cloneUrl", "teamId", "appInstallationId", "creationTime", "_lastModified", "name", "markedDeleted", "userId", "slug", "settings",
5352
}
5453
statement := fmt.Sprintf(`INSERT INTO d_b_project (%s) VALUES ?;`, strings.Join(columns, ", "))
5554
id := uuid.MustParse(obj["id"].(string))

‎components/gitpod-db/src/auth-provider-entry.spec.db.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ describe("AuthProviderEntryDBSpec", async () => {
3838
status: "verified",
3939
type: "GitHub",
4040
oauthRevision: undefined,
41-
deleted: false,
4241
...ap,
4342
oauth: {
4443
callBackUrl: "example.org/some/callback",
@@ -74,7 +73,7 @@ describe("AuthProviderEntryDBSpec", async () => {
7473
await db.storeAuthProvider(ap2, false);
7574

7675
const all = await db.findAllHosts();
77-
expect(all, "findAllHosts([])").to.deep.equal(["foo", "bar"]);
76+
expect(all.sort(), "findAllHosts([])").to.deep.equal(["foo", "bar"].sort());
7877
});
7978

8079
it("should oauthRevision", async () => {

‎components/gitpod-db/src/tables.ts

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -62,55 +62,18 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider
6262
primaryKeys: ["id"],
6363
timeColumn: "_lastModified",
6464
},
65-
{
66-
name: "d_b_token_entry",
67-
primaryKeys: ["uid"],
68-
deletionColumn: "deleted",
69-
timeColumn: "_lastModified",
70-
},
71-
{
72-
name: "d_b_gitpod_token",
73-
primaryKeys: ["tokenHash"],
74-
deletionColumn: "deleted",
75-
timeColumn: "_lastModified",
76-
dependencies: ["d_b_user"],
77-
},
7865
{
7966
name: "d_b_one_time_secret",
8067
primaryKeys: ["id"],
8168
deletionColumn: "deleted",
8269
timeColumn: "_lastModified",
8370
},
84-
{
85-
name: "d_b_auth_provider_entry",
86-
primaryKeys: ["id"],
87-
deletionColumn: "deleted",
88-
timeColumn: "_lastModified",
89-
},
90-
{
91-
name: "d_b_team",
92-
primaryKeys: ["id"],
93-
deletionColumn: "deleted",
94-
timeColumn: "_lastModified",
95-
},
96-
{
97-
name: "d_b_team_membership",
98-
primaryKeys: ["id"],
99-
deletionColumn: "deleted",
100-
timeColumn: "_lastModified",
101-
},
10271
{
10372
name: "d_b_team_membership_invite",
10473
primaryKeys: ["id"],
10574
deletionColumn: "deleted",
10675
timeColumn: "_lastModified",
10776
},
108-
{
109-
name: "d_b_project",
110-
primaryKeys: ["id"],
111-
deletionColumn: "deleted",
112-
timeColumn: "_lastModified",
113-
},
11477
{
11578
name: "d_b_project_env_var",
11679
primaryKeys: ["id", "projectId"],
@@ -123,18 +86,6 @@ export class GitpodTableDescriptionProvider implements TableDescriptionProvider
12386
deletionColumn: "deleted",
12487
timeColumn: "_lastModified",
12588
},
126-
{
127-
name: "d_b_project_usage",
128-
primaryKeys: ["projectId"],
129-
deletionColumn: "deleted",
130-
timeColumn: "_lastModified",
131-
},
132-
{
133-
name: "d_b_user_ssh_public_key",
134-
primaryKeys: ["id"],
135-
deletionColumn: "deleted",
136-
timeColumn: "_lastModified",
137-
},
13889
{
13990
name: "d_b_stripe_customer",
14091
primaryKeys: ["stripeCustomerId"],

‎components/gitpod-db/src/typeorm/auth-provider-entry-db-impl.ts

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
4646
[id],
4747
);
4848

49-
// 2. then mark as deleted
49+
// 2. then delete
5050
const repo = await this.getAuthProviderRepo();
5151
await repo.delete({ id });
5252
}
@@ -55,7 +55,7 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
5555
exceptOAuthRevisions = exceptOAuthRevisions.filter((r) => r !== ""); // never filter out '' which means "undefined" in the DB
5656

5757
const repo = await this.getAuthProviderRepo();
58-
let query = repo.createQueryBuilder("auth_provider").where("auth_provider.deleted != true");
58+
let query = repo.createQueryBuilder("auth_provider");
5959
if (exceptOAuthRevisions.length > 0) {
6060
query = query.andWhere("auth_provider.oauthRevision NOT IN (:...exceptOAuthRevisions)", {
6161
exceptOAuthRevisions,
@@ -68,18 +68,15 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
6868
const hostField: keyof DBAuthProviderEntry = "host";
6969

7070
const repo = await this.getAuthProviderRepo();
71-
const query = repo.createQueryBuilder("auth_provider").select(hostField).where("auth_provider.deleted != true");
71+
const query = repo.createQueryBuilder("auth_provider").select(hostField);
7272
const result = (await query.execute()) as Pick<DBAuthProviderEntry, "host">[];
7373
// HINT: host is expected to be lower case
7474
return result.map((r) => r.host?.toLowerCase()).filter((h) => !!h);
7575
}
7676

7777
async findByHost(host: string): Promise<AuthProviderEntry | undefined> {
7878
const repo = await this.getAuthProviderRepo();
79-
const query = repo
80-
.createQueryBuilder("auth_provider")
81-
.where(`auth_provider.host = :host`, { host })
82-
.andWhere("auth_provider.deleted != true");
79+
const query = repo.createQueryBuilder("auth_provider").where(`auth_provider.host = :host`, { host });
8380
return query.getOne();
8481
}
8582

@@ -93,17 +90,15 @@ export class AuthProviderEntryDBImpl implements AuthProviderEntryDB {
9390
const query = repo
9491
.createQueryBuilder("auth_provider")
9592
.where(`auth_provider.ownerId = :ownerId`, { ownerId })
96-
.andWhere("(auth_provider.organizationId IS NULL OR auth_provider.organizationId = '')")
97-
.andWhere("auth_provider.deleted != true");
93+
.andWhere("(auth_provider.organizationId IS NULL OR auth_provider.organizationId = '')");
9894
return query.getMany();
9995
}
10096

10197
async findByOrgId(organizationId: string): Promise<AuthProviderEntry[]> {
10298
const repo = await this.getAuthProviderRepo();
10399
const query = repo
104100
.createQueryBuilder("auth_provider")
105-
.where(`auth_provider.organizationId = :organizationId`, { organizationId })
106-
.andWhere("auth_provider.deleted != true");
101+
.where(`auth_provider.organizationId = :organizationId`, { organizationId });
107102
return query.getMany();
108103
}
109104

‎components/gitpod-db/src/typeorm/entity/db-auth-provider-entry.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,4 @@ export class DBAuthProviderEntry implements AuthProviderEntry {
4545
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED,
4646
})
4747
oauthRevision?: string;
48-
49-
@Column()
50-
deleted?: boolean;
5148
}

‎components/gitpod-db/src/typeorm/entity/db-gitpod-token.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,4 @@ export class DBGitpodToken implements GitpodToken {
3232

3333
@Column()
3434
created: string;
35-
36-
@Column()
37-
deleted?: boolean;
3835
}

‎components/gitpod-db/src/typeorm/entity/db-project-usage.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,4 @@ export class DBProjectUsage {
1919

2020
@Column("varchar")
2121
lastWorkspaceStart: string;
22-
23-
// This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
24-
@Column()
25-
deleted: boolean;
2622
}

‎components/gitpod-db/src/typeorm/entity/db-project.ts

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66

77
import { Entity, Column, PrimaryColumn, Index } from "typeorm";
88
import { TypeORM } from "../typeorm";
9-
import { ProjectSettings } from "@gitpod/gitpod-protocol";
9+
import { Project, ProjectSettings } from "@gitpod/gitpod-protocol";
1010
import { Transformer } from "../transformer";
1111

1212
@Entity()
1313
// on DB but not Typeorm: @Index("ind_lastModified", ["_lastModified"]) // DBSync
14-
export class DBProject {
14+
export class DBProject implements Project {
1515
@PrimaryColumn(TypeORM.UUID_COLUMN_TYPE)
1616
id: string;
1717

@@ -38,10 +38,6 @@ export class DBProject {
3838
@Column("varchar")
3939
creationTime: string;
4040

41-
// This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
42-
@Column()
43-
deleted: boolean;
44-
4541
@Column()
4642
markedDeleted: boolean;
4743
}

‎components/gitpod-db/src/typeorm/entity/db-team-membership.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,4 @@ export class DBTeamMembership {
2727

2828
@Column("varchar")
2929
creationTime: string;
30-
31-
// This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
32-
@Column()
33-
deleted: boolean;
3430
}

‎components/gitpod-db/src/typeorm/entity/db-team.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,4 @@ export class DBTeam implements Team {
2525

2626
@Column()
2727
markedDeleted?: boolean;
28-
29-
// This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
30-
@Column()
31-
deleted: boolean;
3228
}

‎components/gitpod-db/src/typeorm/entity/db-token-entry.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,4 @@ export class DBTokenEntry implements TokenEntry {
4242
),
4343
})
4444
token: Token;
45-
46-
@Column()
47-
deleted?: boolean;
4845
}

‎components/gitpod-db/src/typeorm/entity/db-user-ssh-public-key.ts

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,4 @@ export class DBUserSshPublicKey implements UserSSHPublicKey {
4949
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED,
5050
})
5151
lastUsedTime?: string;
52-
53-
// This column triggers the periodic deleter deletion mechanism. It's not intended for public consumption.
54-
@Column()
55-
deleted: boolean;
5652
}

‎components/gitpod-db/src/typeorm/migration/1695810605786-TeamDropDeleted.ts

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/**
2+
* Copyright (c) 2023 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { MigrationInterface, QueryRunner } from "typeorm";
8+
import { columnExists } from "./helper/helper";
9+
10+
const TABLE_NAME = "d_b_team";
11+
const COLUMN_NAME = "deleted";
12+
13+
export class TeamDropDeleted1695810605786 implements MigrationInterface {
14+
public async up(queryRunner: QueryRunner): Promise<void> {
15+
if (await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME)) {
16+
await queryRunner.query(`ALTER TABLE \`${TABLE_NAME}\` DROP COLUMN \`${COLUMN_NAME}\``);
17+
}
18+
}
19+
20+
public async down(queryRunner: QueryRunner): Promise<void> {
21+
if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) {
22+
await queryRunner.query(
23+
`ALTER TABLE \`${TABLE_NAME}\` ADD COLUMN \`${COLUMN_NAME}\` tinyint(4) NOT NULL DEFAULT '0'`,
24+
);
25+
}
26+
}
27+
}

‎components/gitpod-db/src/typeorm/migration/1695817445588-ProjectDropDeleted.ts

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/**
2+
* Copyright (c) 2023 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { MigrationInterface, QueryRunner } from "typeorm";
8+
import { columnExists } from "./helper/helper";
9+
10+
const TABLE_NAME = "d_b_project";
11+
const COLUMN_NAME = "deleted";
12+
13+
export class ProjectDropDeleted1695817445588 implements MigrationInterface {
14+
public async up(queryRunner: QueryRunner): Promise<void> {
15+
if (await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME)) {
16+
await queryRunner.query(`ALTER TABLE \`${TABLE_NAME}\` DROP COLUMN \`${COLUMN_NAME}\``);
17+
}
18+
}
19+
20+
public async down(queryRunner: QueryRunner): Promise<void> {
21+
if (!(await columnExists(queryRunner, TABLE_NAME, COLUMN_NAME))) {
22+
await queryRunner.query(
23+
`ALTER TABLE \`${TABLE_NAME}\` ADD COLUMN \`${COLUMN_NAME}\` tinyint(4) NOT NULL DEFAULT '0'`,
24+
);
25+
}
26+
}
27+
}

‎components/gitpod-db/src/typeorm/migration/1695819413747-DropTableUserStorageResource.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/**
2+
* Copyright (c) 2023 Gitpod GmbH. All rights reserved.
3+
* Licensed under the GNU Affero General Public License (AGPL).
4+
* See License.AGPL.txt in the project root for license information.
5+
*/
6+
7+
import { MigrationInterface, QueryRunner } from "typeorm";
8+
9+
export class DropTableUserStorageResource1695819413747 implements MigrationInterface {
10+
public async up(queryRunner: QueryRunner): Promise<void> {
11+
await queryRunner.query("DROP TABLE IF EXISTS `d_b_user_storage_resource`");
12+
}
13+
14+
public async down(queryRunner: QueryRunner): Promise<void> {}
15+
}

0 commit comments

Comments
 (0)