Skip to content

Commit 30f4a34

Browse files
exdxexdx
committed
fix: support creating v1beta CRDs to avoid data loss during conversion to v1. Due to data loss during client side conversions OLM will support two different paths for v1 and v1beta1 CRDs.
1 parent ed17b12 commit 30f4a34

File tree

7 files changed

+364
-118
lines changed

7 files changed

+364
-118
lines changed

‎pkg/controller/operators/catalog/operator.go

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
rbacv1 "k8s.io/api/rbac/v1"
1818
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
1919
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
20+
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
2021
"k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
2122
extinf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -1315,7 +1316,21 @@ func GetCRDV1VersionsMap(crd *apiextensionsv1.CustomResourceDefinition) map[stri
13151316
}
13161317

13171318
// Ensure all existing served versions are present in new CRD
1318-
func EnsureCRDVersions(oldCRD *apiextensionsv1.CustomResourceDefinition, newCRD *apiextensionsv1.CustomResourceDefinition) error {
1319+
func GetCRDV1Beta1VersionsMap(crd *apiextensionsv1beta1.CustomResourceDefinition) map[string]struct{} {
1320+
versionsMap := map[string]struct{}{}
1321+
1322+
for _, version := range crd.Spec.Versions {
1323+
versionsMap[version.Name] = struct{}{}
1324+
}
1325+
if crd.Spec.Version != "" {
1326+
versionsMap[crd.Spec.Version] = struct{}{}
1327+
}
1328+
1329+
return versionsMap
1330+
}
1331+
1332+
// Ensure all existing served versions are present in new CRD
1333+
func EnsureV1CRDVersions(oldCRD *apiextensionsv1.CustomResourceDefinition, newCRD *apiextensionsv1.CustomResourceDefinition) error {
13191334
newCRDVersions := GetCRDV1VersionsMap(newCRD)
13201335

13211336
for _, oldVersion := range oldCRD.Spec.Versions {
@@ -1329,6 +1344,27 @@ func EnsureCRDVersions(oldCRD *apiextensionsv1.CustomResourceDefinition, newCRD
13291344
return nil
13301345
}
13311346

1347+
// Ensure all existing served versions are present in new CRD
1348+
func EnsureV1Beta1CRDVersions(oldCRD *apiextensionsv1beta1.CustomResourceDefinition, newCRD *apiextensionsv1beta1.CustomResourceDefinition) error {
1349+
newCRDVersions := GetCRDV1Beta1VersionsMap(newCRD)
1350+
1351+
for _, oldVersion := range oldCRD.Spec.Versions {
1352+
if oldVersion.Served {
1353+
_, ok := newCRDVersions[oldVersion.Name]
1354+
if !ok {
1355+
return fmt.Errorf("New CRD (%s) must contain existing served versions (%s)", oldCRD.Name, oldVersion.Name)
1356+
}
1357+
}
1358+
}
1359+
if oldCRD.Spec.Version != "" {
1360+
_, ok := newCRDVersions[oldCRD.Spec.Version]
1361+
if !ok {
1362+
return fmt.Errorf("New CRD (%s) must contain existing version (%s)", oldCRD.Name, oldCRD.Spec.Version)
1363+
}
1364+
}
1365+
return nil
1366+
}
1367+
13321368
// Validate all existing served versions against new CRD's validation (if changed)
13331369
func validateV1CRDCompatibility(dynamicClient dynamic.Interface, oldCRD *apiextensionsv1.CustomResourceDefinition, newCRD *apiextensionsv1.CustomResourceDefinition) error {
13341370
logrus.Debugf("Comparing %#v to %#v", oldCRD.Spec.Versions, newCRD.Spec.Versions)
@@ -1364,6 +1400,38 @@ func validateV1CRDCompatibility(dynamicClient dynamic.Interface, oldCRD *apiexte
13641400
return nil
13651401
}
13661402

1403+
// Validate all existing served versions against new CRD's validation (if changed)
1404+
func validateV1Beta1CRDCompatibility(dynamicClient dynamic.Interface, oldCRD *apiextensionsv1beta1.CustomResourceDefinition, newCRD *apiextensionsv1beta1.CustomResourceDefinition) error {
1405+
logrus.Debugf("Comparing %#v to %#v", oldCRD.Spec.Validation, newCRD.Spec.Validation)
1406+
// If validation schema is unchanged, return right away
1407+
if reflect.DeepEqual(oldCRD.Spec.Validation, newCRD.Spec.Validation) {
1408+
return nil
1409+
}
1410+
convertedCRD := &apiextensions.CustomResourceDefinition{}
1411+
if err := apiextensionsv1beta1.Convert_v1beta1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(newCRD, convertedCRD, nil); err != nil {
1412+
return err
1413+
}
1414+
for _, version := range oldCRD.Spec.Versions {
1415+
if !version.Served {
1416+
gvr := schema.GroupVersionResource{Group: oldCRD.Spec.Group, Version: version.Name, Resource: oldCRD.Spec.Names.Plural}
1417+
err := validateExistingCRs(dynamicClient, gvr, convertedCRD)
1418+
if err != nil {
1419+
return err
1420+
}
1421+
}
1422+
}
1423+
1424+
if oldCRD.Spec.Version != "" {
1425+
gvr := schema.GroupVersionResource{Group: oldCRD.Spec.Group, Version: oldCRD.Spec.Version, Resource: oldCRD.Spec.Names.Plural}
1426+
err := validateExistingCRs(dynamicClient, gvr, convertedCRD)
1427+
if err != nil {
1428+
return err
1429+
}
1430+
}
1431+
logrus.Debugf("Successfully validated CRD %s\n", newCRD.Name)
1432+
return nil
1433+
}
1434+
13671435
func validateExistingCRs(dynamicClient dynamic.Interface, gvr schema.GroupVersionResource, newCRD *apiextensions.CustomResourceDefinition) error {
13681436
// make dynamic client
13691437
crList, err := dynamicClient.Resource(gvr).List(context.TODO(), metav1.ListOptions{})
@@ -1410,6 +1478,28 @@ func removeDeprecatedV1StoredVersions(oldCRD *apiextensionsv1.CustomResourceDefi
14101478
}
14111479
}
14121480

1481+
func removeDeprecatedV1Beta1StoredVersions(oldCRD *apiextensionsv1beta1.CustomResourceDefinition, newCRD *apiextensionsv1beta1.CustomResourceDefinition) []string {
1482+
// StoredVersions requires to have at least one version.
1483+
if len(oldCRD.Status.StoredVersions) <= 1 {
1484+
return nil
1485+
}
1486+
1487+
newStoredVersions := []string{}
1488+
newCRDVersions := GetCRDV1Beta1VersionsMap(newCRD)
1489+
for _, v := range oldCRD.Status.StoredVersions {
1490+
_, ok := newCRDVersions[v]
1491+
if ok {
1492+
newStoredVersions = append(newStoredVersions, v)
1493+
}
1494+
}
1495+
1496+
if len(newStoredVersions) < 1 {
1497+
return nil
1498+
} else {
1499+
return newStoredVersions
1500+
}
1501+
}
1502+
14131503
// ExecutePlan applies a planned InstallPlan to a namespace.
14141504
func (o *Operator) ExecutePlan(plan *v1alpha1.InstallPlan) error {
14151505
if plan.Status.Phase != v1alpha1.InstallPlanPhaseInstalling {
@@ -1436,7 +1526,7 @@ func (o *Operator) ExecutePlan(plan *v1alpha1.InstallPlan) error {
14361526
}
14371527

14381528
ensurer := newStepEnsurer(kubeclient, crclient, dynamicClient)
1439-
b := newBuilder(kubeclient, dynamicClient, o.csvProvidedAPIsIndexer)
1529+
b := newBuilder(kubeclient, dynamicClient, o.csvProvidedAPIsIndexer, o.logger)
14401530

14411531
for i, step := range plan.Status.Plan {
14421532
doStep := true

‎pkg/controller/operators/catalog/step.go

Lines changed: 139 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@ import (
1313
errorwrap "github.com/pkg/errors"
1414
logger "github.com/sirupsen/logrus"
1515
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
16-
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
16+
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
17+
apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
18+
apiextensionsv1beta1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1"
19+
1720
k8serrors "k8s.io/apimachinery/pkg/api/errors"
1821
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1922
)
@@ -36,13 +39,16 @@ type builder struct {
3639
opclient operatorclient.ClientInterface
3740
dynamicClient dynamic.Interface
3841
csvToProvidedAPIs map[string]cache.Indexer
42+
logger logger.FieldLogger
3943
}
4044

41-
func newBuilder(opclient operatorclient.ClientInterface, dynamicClient dynamic.Interface, csvToProvidedAPIs map[string]cache.Indexer) *builder {
45+
func newBuilder(opclient operatorclient.ClientInterface, dynamicClient dynamic.Interface, csvToProvidedAPIs map[string]cache.Indexer,
46+
logger logger.FieldLogger) *builder {
4247
return &builder{
4348
opclient: opclient,
4449
dynamicClient: dynamicClient,
4550
csvToProvidedAPIs: csvToProvidedAPIs,
51+
logger: logger,
4652
}
4753
}
4854

@@ -57,23 +63,32 @@ func (n notSupportedStepperErr) Error() string {
5763
// step is a factory that creates StepperFuncs based on the Kind provided and the install plan step.
5864
func (b *builder) create(step *v1alpha1.Step) (Stepper, error) {
5965
kind := step.Resource.Kind
66+
version, err := crdlib.Version(&step.Resource.Manifest)
67+
if err != nil {
68+
return nil, err
69+
}
70+
6071
switch kind {
6172
case crdKind:
62-
return b.NewCRDStep(step.Resource.Manifest, b.opclient.ApiextensionsInterface(), step.Status, step.Resource.Name), nil
63-
default:
64-
return nil, notSupportedStepperErr{fmt.Sprintf("stepper interface does not support %s", kind)}
73+
switch version {
74+
case crdlib.V1Version:
75+
return b.NewCRDV1Step(b.opclient.ApiextensionsInterface().ApiextensionsV1(), step), nil
76+
case crdlib.V1Beta1Version:
77+
return b.NewCRDV1Beta1Step(b.opclient.ApiextensionsInterface().ApiextensionsV1beta1(), step), nil
78+
}
6579
}
80+
return nil, notSupportedStepperErr{fmt.Sprintf("stepper interface does not support %s", kind)}
6681
}
6782

68-
func (b *builder) NewCRDStep(manifest string, client clientset.Interface, status v1alpha1.StepStatus, name string) StepperFunc {
83+
func (b *builder) NewCRDV1Step(client apiextensionsv1client.ApiextensionsV1Interface, step *v1alpha1.Step) StepperFunc {
6984
return func() (v1alpha1.StepStatus, error) {
70-
switch status {
85+
switch step.Status {
7186
case v1alpha1.StepStatusPresent:
7287
return v1alpha1.StepStatusPresent, nil
7388
case v1alpha1.StepStatusCreated:
7489
return v1alpha1.StepStatusCreated, nil
7590
case v1alpha1.StepStatusWaitingForAPI:
76-
crd, err := client.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), name, metav1.GetOptions{})
91+
crd, err := client.CustomResourceDefinitions().Get(context.TODO(), step.Resource.Name, metav1.GetOptions{})
7792
if err != nil {
7893
if k8serrors.IsNotFound(err) {
7994
return v1alpha1.StepStatusNotPresent, nil
@@ -98,61 +113,161 @@ func (b *builder) NewCRDStep(manifest string, client clientset.Interface, status
98113
return v1alpha1.StepStatusCreated, nil
99114
}
100115
case v1alpha1.StepStatusUnknown, v1alpha1.StepStatusNotPresent:
101-
crd, err := crdlib.Serialize(manifest)
116+
crd, err := crdlib.UnmarshalV1(step.Resource.Manifest)
102117
if err != nil {
103118
return v1alpha1.StepStatusUnknown, err
104119
}
120+
b.logger.Debugf("creating v1 CRD %#v", crd)
105121

106-
_, err = client.ApiextensionsV1().CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
107-
if k8serrors.IsAlreadyExists(err) {
108-
currentCRD, _ := client.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
109-
// Compare 2 CRDs to see if it needs to be updatetd
110-
if crdlib.NotEqual(currentCRD, crd) {
122+
_, createError := client.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
123+
if k8serrors.IsAlreadyExists(createError) {
124+
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
125+
// Compare 2 CRDs to see if it needs to be updated
126+
logger.Debugf("\n current crd: %#v \n new crd: %#v \n", currentCRD, crd)
127+
if crdlib.V1NotEqual(currentCRD, crd) {
111128
// Verify CRD ownership, only attempt to update if
112129
// CRD has only one owner
113130
// Example: provided=database.coreos.com/v1alpha1/EtcdCluster
114-
matchedCSV, err := index.CRDProviderNames(b.csvToProvidedAPIs, crd)
131+
matchedCSV, err := index.V1CRDProviderNames(b.csvToProvidedAPIs, crd)
115132
if err != nil {
116-
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error find matched CSV: %s", name)
133+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error find matched CSV: %s", step.Resource.Name)
117134
}
118135
crd.SetResourceVersion(currentCRD.GetResourceVersion())
119136
if len(matchedCSV) == 1 {
120137
logger.Debugf("Found one owner for CRD %v", crd)
121138
} else if len(matchedCSV) > 1 {
122139
logger.Debugf("Found multiple owners for CRD %v", crd)
123140

124-
err := EnsureCRDVersions(currentCRD, crd)
141+
err := EnsureV1CRDVersions(currentCRD, crd)
125142
if err != nil {
126-
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error missing existing CRD version(s) in new CRD: %s", name)
143+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error missing existing CRD version(s) in new CRD: %s", step.Resource.Name)
127144
}
128145

129146
if err = validateV1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
130-
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error validating existing CRs agains new CRD's schema: %s", name)
147+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error validating existing CRs agains new CRD's schema: %s", step.Resource.Name)
131148
}
132149
}
133150
// Remove deprecated version in CRD storedVersions
134151
storeVersions := removeDeprecatedV1StoredVersions(currentCRD, crd)
135152
if storeVersions != nil {
136153
currentCRD.Status.StoredVersions = storeVersions
137-
resultCRD, err := client.ApiextensionsV1().CustomResourceDefinitions().UpdateStatus(context.TODO(), currentCRD, metav1.UpdateOptions{})
154+
resultCRD, err := client.CustomResourceDefinitions().UpdateStatus(context.TODO(), currentCRD, metav1.UpdateOptions{})
138155
if err != nil {
139-
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD's status: %s", name)
156+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD's status: %s", step.Resource.Name)
140157
}
141158
crd.SetResourceVersion(resultCRD.GetResourceVersion())
142159
}
143160
// Update CRD to new version
144-
_, err = client.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
161+
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
145162
if err != nil {
146-
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD: %s", name)
163+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD: %s", step.Resource.Name)
147164
}
148165
}
149166
// If it already existed, mark the step as Present.
150167
// they were equal - mark CRD as present
151168
return v1alpha1.StepStatusPresent, nil
152-
} else if err != nil {
169+
} else if createError != nil {
153170
// Unexpected error creating the CRD.
171+
return v1alpha1.StepStatusUnknown, createError
172+
}
173+
// If no error occured, make sure to wait for the API to become available.
174+
return v1alpha1.StepStatusWaitingForAPI, nil
175+
}
176+
return v1alpha1.StepStatusUnknown, nil
177+
}
178+
}
179+
180+
func (b *builder) NewCRDV1Beta1Step(client apiextensionsv1beta1client.ApiextensionsV1beta1Interface, step *v1alpha1.Step) StepperFunc {
181+
return func() (v1alpha1.StepStatus, error) {
182+
switch step.Status {
183+
case v1alpha1.StepStatusPresent:
184+
return v1alpha1.StepStatusPresent, nil
185+
case v1alpha1.StepStatusCreated:
186+
return v1alpha1.StepStatusCreated, nil
187+
case v1alpha1.StepStatusWaitingForAPI:
188+
crd, err := client.CustomResourceDefinitions().Get(context.TODO(), step.Resource.Name, metav1.GetOptions{})
189+
if err != nil {
190+
if k8serrors.IsNotFound(err) {
191+
return v1alpha1.StepStatusNotPresent, nil
192+
} else {
193+
return v1alpha1.StepStatusNotPresent, errorwrap.Wrapf(err, "error finding the %s CRD", crd.Name)
194+
}
195+
}
196+
established, namesAccepted := false, false
197+
for _, cdt := range crd.Status.Conditions {
198+
switch cdt.Type {
199+
case apiextensionsv1beta1.Established:
200+
if cdt.Status == apiextensionsv1beta1.ConditionTrue {
201+
established = true
202+
}
203+
case apiextensionsv1beta1.NamesAccepted:
204+
if cdt.Status == apiextensionsv1beta1.ConditionTrue {
205+
namesAccepted = true
206+
}
207+
}
208+
}
209+
if established && namesAccepted {
210+
return v1alpha1.StepStatusCreated, nil
211+
}
212+
case v1alpha1.StepStatusUnknown, v1alpha1.StepStatusNotPresent:
213+
crd, err := crdlib.UnmarshalV1Beta1(step.Resource.Manifest)
214+
if err != nil {
154215
return v1alpha1.StepStatusUnknown, err
155216
}
217+
b.logger.Debugf("creating v1beta1 CRD %#v", crd)
218+
219+
_, createError := client.CustomResourceDefinitions().Create(context.TODO(), crd, metav1.CreateOptions{})
220+
if k8serrors.IsAlreadyExists(createError) {
221+
currentCRD, _ := client.CustomResourceDefinitions().Get(context.TODO(), crd.GetName(), metav1.GetOptions{})
222+
// Compare 2 CRDs to see if it needs to be updated
223+
logger.Debugf("\n current crd: %#v \n new crd: %#v \n", currentCRD, crd)
224+
if crdlib.V1Beta1NotEqual(currentCRD, crd) {
225+
b.logger.Debugf("not equal")
226+
// Verify CRD ownership, only attempt to update if
227+
// CRD has only one owner
228+
// Example: provided=database.coreos.com/v1alpha1/EtcdCluster
229+
matchedCSV, err := index.V1Beta1CRDProviderNames(b.csvToProvidedAPIs, crd)
230+
if err != nil {
231+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error find matched CSV: %s", step.Resource.Name)
232+
}
233+
crd.SetResourceVersion(currentCRD.GetResourceVersion())
234+
if len(matchedCSV) == 1 {
235+
logger.Debugf("Found one owner for CRD %v", crd)
236+
} else if len(matchedCSV) > 1 {
237+
logger.Debugf("Found multiple owners for CRD %v", crd)
238+
239+
err := EnsureV1Beta1CRDVersions(currentCRD, crd)
240+
if err != nil {
241+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error missing existing CRD version(s) in new CRD: %s", step.Resource.Name)
242+
}
243+
244+
if err = validateV1Beta1CRDCompatibility(b.dynamicClient, currentCRD, crd); err != nil {
245+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error validating existing CRs agains new CRD's schema: %s", step.Resource.Name)
246+
}
247+
}
248+
// Remove deprecated version in CRD storedVersions
249+
storeVersions := removeDeprecatedV1Beta1StoredVersions(currentCRD, crd)
250+
if storeVersions != nil {
251+
currentCRD.Status.StoredVersions = storeVersions
252+
resultCRD, err := client.CustomResourceDefinitions().UpdateStatus(context.TODO(), currentCRD, metav1.UpdateOptions{})
253+
if err != nil {
254+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD's status: %s", step.Resource.Name)
255+
}
256+
crd.SetResourceVersion(resultCRD.GetResourceVersion())
257+
}
258+
// Update CRD to new version
259+
_, err = client.CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{})
260+
if err != nil {
261+
return v1alpha1.StepStatusUnknown, errorwrap.Wrapf(err, "error updating CRD: %s", step.Resource.Name)
262+
}
263+
}
264+
// If it already existed, mark the step as Present.
265+
// they were equal - mark CRD as present
266+
return v1alpha1.StepStatusPresent, nil
267+
} else if createError != nil {
268+
// Unexpected error creating the CRD.
269+
return v1alpha1.StepStatusUnknown, createError
270+
}
156271
// If no error occured, make sure to wait for the API to become available.
157272
return v1alpha1.StepStatusWaitingForAPI, nil
158273
}

0 commit comments

Comments
 (0)