Support EBS gp2 to gp3 migration on sync for below 1tb volumes (#1242)

* initial commit for gp3 migration.

* Default volume migration done.

* Added Gomock and one test case with mock.

* Dep update.

* more changes for code gen.

* push fake package.

* Rename var.

* Changes to Makefile and return value.

* Macke mocks phony due to overlap in foldername.

* Learning as one goes. Initialize map.

* Wrong toggle.

* Expect modify call.

* Fix mapping of ids in test.

* Fix volume id.

* volume ids.

* Fixing test setup. Late night...

* create all pvs.

* Fix test case config.

* store volumes and compare.

* More logs.

* Logging of migration action.

* Ensure to log errors.

* Log warning if modify failed, e.g. due to ebs volume state.

* Add more output.

* Skip local e2e tests.

* Reflect k8s volume id in test data. Extract aws volume id from k8s value.

* Finalizing ebs migration.

* More logs. describe fails.

* Fix non existing fields in gp2 discovery.

* Remove nothing to do flag for migration.

* Final commit for migration.

* add new options to all places

Co-authored-by: Felix Kunde <[email protected]>

.github/workflows/run_e2e.yaml

@@ -16,7 +16,7 @@ jobs:
       with:
           go-version: "^1.15.5"
     - name: Make dependencies
-      run: make deps
+      run: make deps mocks
     - name: Compile
       run: make linux
     - name: Run unit tests

.github/workflows/run_tests.yaml

@@ -16,7 +16,7 @@ jobs:
       with:
           go-version: "^1.15.5"
     - name: Make dependencies
-      run: make deps
+      run: make deps mocks
     - name: Compile
       run: make linux
     - name: Run unit tests

.gitignore

@@ -98,3 +98,5 @@ e2e/manifests
 # Translations
 *.mo
 *.pot
+
+mocks

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean local test linux macos docker push scm-source.json e2e
+.PHONY: clean local test linux macos mocks docker push scm-source.json e2e
 
 BINARY ?= postgres-operator
 BUILD_FLAGS ?= -v
@@ -81,9 +81,12 @@ push:
 scm-source.json: .git
 	echo '{\n "url": "git:$(GITURL)",\n "revision": "$(GITHEAD)",\n "author": "$(USER)",\n "status": "$(GITSTATUS)"\n}' > scm-source.json
 
+mocks:
+	GO111MODULE=on go generate ./...
+
 tools:
-	GO111MODULE=on go get -u honnef.co/go/tools/cmd/staticcheck
 	GO111MODULE=on go get k8s.io/[email protected]
+	GO111MODULE=on go get github.com/golang/mock/[email protected]
 	GO111MODULE=on go mod tidy
 
 fmt:

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -292,6 +292,10 @@ spec:
                     type: string
                   aws_region:
                     type: string
+                  enable_ebs_gp3_migration:
+                    type: boolean
+                  enable_ebs_gp3_migration_max_size:
+                    type: integer
                   gcp_credentials:
                     type: string
                   kube_iam_role:

charts/postgres-operator/values-crd.yaml

@@ -219,6 +219,11 @@ configAwsOrGcp:
   # AWS region used to store ESB volumes
   aws_region: eu-central-1
 
+  # enable automatic migration on AWS from gp2 to gp3 volumes
+  enable_ebs_gp3_migration: false
+  # defines maximum volume size in GB until which auto migration happens
+  # enable_ebs_gp3_migration_max_size: 1000
+
   # GCP credentials that will be used by the operator / pods
   # gcp_credentials: ""
 

charts/postgres-operator/values.yaml

@@ -211,6 +211,14 @@ configAwsOrGcp:
   # AWS region used to store ESB volumes
   aws_region: eu-central-1
 
+  # enable automatic migration on AWS from gp2 to gp3 volumes
+  enable_ebs_gp3_migration: "false"
+  # defines maximum volume size in GB until which auto migration happens
+  # enable_ebs_gp3_migration_max_size: 1000
+
+  # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable
+  # gcp_credentials: ""
+
   # AWS IAM role to supply in the iam.amazonaws.com/role annotation of Postgres pods
   # kube_iam_role: ""
 
@@ -223,9 +231,6 @@ configAwsOrGcp:
   # GCS bucket to use for shipping WAL segments with WAL-E
   # wal_gs_bucket: ""
 
-  # GCP credentials for setting the GOOGLE_APPLICATION_CREDNETIALS environment variable
-  # gcp_credentials: ""
-
 # configure K8s cron job managed by the operator
 configLogicalBackup:
   # image for pods of the logical backup job (example runs pg_dumpall)

delivery.yaml

@@ -32,7 +32,7 @@ pipeline:
               IMAGE=registry-write.opensource.zalan.do/acid/postgres-operator-test
             fi
             export IMAGE
-            make deps docker
+            make deps mocks docker
         - desc: 'Run unit tests'
           cmd: |
             export PATH=$PATH:$HOME/go/bin

docs/reference/operator_parameters.md

@@ -518,10 +518,22 @@ yet officially supported.
   AWS region used to store EBS volumes. The default is `eu-central-1`.
 
 * **additional_secret_mount**
-  Additional Secret (aws or gcp credentials) to mount in the pod. The default is empty.
+  Additional Secret (aws or gcp credentials) to mount in the pod.
+  The default is empty.
 
 * **additional_secret_mount_path**
-  Path to mount the above Secret in the filesystem of the container(s). The default is empty.
+  Path to mount the above Secret in the filesystem of the container(s).
+  The default is empty.
+
+* **enable_ebs_gp3_migration**
+  enable automatic migration on AWS from gp2 to gp3 volumes, that are smaller
+  than the configured max size (see below). This ignores that EBS gp3 is by
+  default only 125 MB/sec vs 250 MB/sec for gp2 >= 333GB.
+  The default is `false`.
+
+* **enable_ebs_gp3_migration_max_size**
+  defines the maximum volume size in GB until which auto migration happens.
+  Default is 1000 (1TB) which matches 3000 IOPS.
 
 ## Logical backup
 

go.mod

@@ -1,20 +1,22 @@
 module github.com/zalando/postgres-operator
 
-go 1.14
+go 1.15
 
 require (
-	github.com/aws/aws-sdk-go v1.35.15
-	github.com/lib/pq v1.8.0
+	github.com/aws/aws-sdk-go v1.36.3
+	github.com/golang/mock v1.4.4
+	github.com/lib/pq v1.9.0
 	github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d
 	github.com/r3labs/diff v1.1.0
 	github.com/sirupsen/logrus v1.7.0
-	github.com/stretchr/testify v1.5.1
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/tools v0.0.0-20201121010211-780cb80bd7fb // indirect
-	gopkg.in/yaml.v2 v2.2.8
-	k8s.io/api v0.19.3
+	github.com/stretchr/testify v1.6.1
+	golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c
+	golang.org/x/mod v0.4.0 // indirect
+	golang.org/x/tools v0.0.0-20201207204333-a835c872fcea // indirect
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.19.4
 	k8s.io/apiextensions-apiserver v0.19.3
-	k8s.io/apimachinery v0.19.3
+	k8s.io/apimachinery v0.19.4
 	k8s.io/client-go v0.19.3
-	k8s.io/code-generator v0.19.3
+	k8s.io/code-generator v0.19.4
 )