Skip to content

Commit

Permalink
fix(gcp): Use Cloud Run Jobs to run db migrations (#689)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tjholm
tjholm authored Oct 29, 2024
1 parent 7c112d6 commit da6b0fc
Show file tree
Hide file tree
Showing 7 changed files with 63 additions and 180 deletions.
4 changes: 2 additions & 2 deletions cloud/gcp/deploy/batch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import (
"github.com/nitrictech/nitric/cloud/common/deploy/provider"
deploymentspb "github.com/nitrictech/nitric/core/pkg/proto/deployments/v1"
"github.com/pkg/errors"
"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/projects"
"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/serviceaccount"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/serviceaccount"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"google.golang.org/protobuf/encoding/protojson"
Expand Down
70 changes: 11 additions & 59 deletions cloud/gcp/deploy/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ import (
deploymentspb "github.com/nitrictech/nitric/core/pkg/proto/deployments/v1"
"github.com/pkg/errors"
"github.com/pulumi/pulumi-docker/sdk/v4/go/docker"
"github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/artifactregistry"
workerpool "github.com/pulumi/pulumi-gcp/sdk/v6/go/gcp/cloudbuild"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/apigateway"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/artifactregistry"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/cloudrunv2"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/cloudtasks"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/compute"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/firestore"
Expand All @@ -45,12 +45,10 @@ import (
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/pubsub"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/secretmanager"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/servicenetworking"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/serviceusage"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/sql"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/vpcaccess"
"github.com/pulumi/pulumi-random/sdk/v4/go/random"
"github.com/pulumi/pulumi-std/sdk/go/std"
"github.com/pulumi/pulumi/sdk/v3/go/auto"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"github.com/samber/lo"
Expand Down Expand Up @@ -89,12 +87,11 @@ type NitricGcpPulumiProvider struct {
Queues map[string]*pubsub.Topic
QueueSubscriptions map[string]*pubsub.Subscription
Secrets map[string]*secretmanager.Secret
DatabaseMigrationBuild map[string]*CloudBuild
DatabaseMigrationBuild map[string]*cloudrunv2.Job

BatchServiceAccounts map[string]*GcpIamServiceAccount
masterDb *sql.DatabaseInstance
dbMasterPassword *random.RandomPassword
cloudBuildWorkerPool *workerpool.WorkerPool
privateNetwork *compute.Network
privateSubnet *compute.Subnetwork
vpcConnector *vpcaccess.Connector
Expand All @@ -104,7 +101,7 @@ type NitricGcpPulumiProvider struct {

var _ provider.NitricPulumiProvider = (*NitricGcpPulumiProvider)(nil)

const pulumiGcpVersion = "6.67.1"
const pulumiGcpVersion = "8.6.0"

func (a *NitricGcpPulumiProvider) Config() (auto.ConfigMap, error) {
return auto.ConfigMap{
Expand Down Expand Up @@ -440,7 +437,7 @@ func NewNitricGcpProvider() *NitricGcpPulumiProvider {
Queues: make(map[string]*pubsub.Topic),
QueueSubscriptions: make(map[string]*pubsub.Subscription),
Secrets: make(map[string]*secretmanager.Secret),
DatabaseMigrationBuild: make(map[string]*CloudBuild),
DatabaseMigrationBuild: make(map[string]*cloudrunv2.Job),
}
}

Expand Down Expand Up @@ -488,7 +485,6 @@ func (a *NitricGcpPulumiProvider) createCloudSQLDatabase(ctx *pulumi.Context) er
}

a.privateNetwork, err = compute.NewNetwork(ctx, "nitric-db-private-network", &compute.NetworkArgs{
Name: pulumi.String("nitric-db-private-network"),
Project: pulumi.String(a.GcpConfig.ProjectId),
AutoCreateSubnetworks: pulumi.Bool(false),
})
Expand All @@ -497,7 +493,6 @@ func (a *NitricGcpPulumiProvider) createCloudSQLDatabase(ctx *pulumi.Context) er
}

a.privateSubnet, err = compute.NewSubnetwork(ctx, "nitric-db-subnetwork", &compute.SubnetworkArgs{
Name: pulumi.String("nitric-db-subnetwork"),
IpCidrRange: pulumi.String("10.0.0.0/26"),
Region: pulumi.String(a.Region),
Project: pulumi.String(a.GcpConfig.ProjectId),
Expand All @@ -515,7 +510,6 @@ func (a *NitricGcpPulumiProvider) createCloudSQLDatabase(ctx *pulumi.Context) er
}

privateIpRange, err := compute.NewGlobalAddress(ctx, "nitric-db-ip-range", &compute.GlobalAddressArgs{
Name: pulumi.String("nitric-db-ip-range"),
Project: pulumi.String(a.GcpConfig.ProjectId),
Purpose: pulumi.String("VPC_PEERING"),
AddressType: pulumi.String("INTERNAL"),
Expand All @@ -532,66 +526,24 @@ func (a *NitricGcpPulumiProvider) createCloudSQLDatabase(ctx *pulumi.Context) er
ReservedPeeringRanges: pulumi.StringArray{
privateIpRange.Name,
},
})
DeletionPolicy: pulumi.String("ABANDON"),
}, pulumi.DependsOn([]pulumi.Resource{a.privateSubnet, privateIpRange}))
if err != nil {
return err
}

// Create an explicit VPC connector for the Google Cloud Run VPC connections
// TODO: Remove this in favor of direct VPC egress when fixed
a.vpcConnector, err = vpcaccess.NewConnector(ctx, "db-vpc-connector", &vpcaccess.ConnectorArgs{
IpCidrRange: pulumi.String("10.8.0.0/28"),
Network: a.privateNetwork.ID(),
})
if err != nil {
return err
}

metricUrlEncode, err := std.Urlencode(ctx, &std.UrlencodeArgs{
Input: "cloudbuild.googleapis.com/private_pools",
}, nil)
if err != nil {
return err
}

regionUrlEncode, err := std.Urlencode(ctx, &std.UrlencodeArgs{
Input: "/project/region",
}, nil)
if err != nil {
return err
}

workerPoolQuota, err := serviceusage.NewConsumerQuotaOverride(ctx, "worker-pool-quota", &serviceusage.ConsumerQuotaOverrideArgs{
Project: pulumi.String(a.GcpConfig.ProjectId),
Service: pulumi.String("cloudbuild.googleapis.com"),
Metric: pulumi.String(metricUrlEncode.Result),
Limit: pulumi.String(regionUrlEncode.Result),
OverrideValue: pulumi.String("1"),
Force: pulumi.Bool(true),
Dimensions: pulumi.StringMap{
"region": pulumi.String(a.Region),
},
IpCidrRange: pulumi.String("10.8.0.0/28"),
Network: a.privateNetwork.ID(),
MaxInstances: pulumi.Int(3),
MinInstances: pulumi.Int(2),
})
if err != nil {
return err
}

a.cloudBuildWorkerPool, err = workerpool.NewWorkerPool(ctx, "cloud-build-worker-pool", &workerpool.WorkerPoolArgs{
Name: pulumi.String("cloud-build-worker-pool"),
Location: pulumi.String(a.Region),
WorkerConfig: &workerpool.WorkerPoolWorkerConfigArgs{
DiskSizeGb: pulumi.Int(100),
MachineType: pulumi.String("e2-medium"),
},
NetworkConfig: &workerpool.WorkerPoolNetworkConfigArgs{
PeeredNetwork: a.privateNetwork.ID(),
// PeeredNetworkIpRange: pulumi.String("/29"),
},
}, pulumi.DependsOn([]pulumi.Resource{privateVpcConnection, workerPoolQuota}))
if err != nil {
return err
}

// generate a db cluster random password
a.dbMasterPassword, err = random.NewRandomPassword(ctx, "nitric-db-master-password", &random.RandomPasswordArgs{
Length: pulumi.Int(16),
Expand Down
2 changes: 1 addition & 1 deletion cloud/gcp/deploy/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ func (p *NitricGcpPulumiProvider) Service(ctx *pulumi.Context, parent pulumi.Res

opts = append(opts, pulumi.DependsOn(dependsOn))

serviceTemplate.Annotations = pulumi.ToStringMapOutput(map[string]pulumi.StringOutput{"run.googleapis.com/cloudsql-instances": p.masterDb.ConnectionName})
// serviceTemplate.Annotations = pulumi.ToStringMapOutput(map[string]pulumi.StringOutput{"run.googleapis.com/cloudsql-instances": p.masterDb.ConnectionName})
}

migrationBuilds := lo.Values(p.DatabaseMigrationBuild)
Expand Down
153 changes: 47 additions & 106 deletions cloud/gcp/deploy/sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,18 @@
package deploy

import (
"context"
"crypto/md5" //#nosec G501 -- md5 used only to produce a unique ID from non-sensistive information (policy IDs)
"encoding/hex"
"fmt"
"strings"
"time"

cloudbuild "cloud.google.com/go/cloudbuild/apiv1/v2"
"cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb"
"github.com/avast/retry-go"
"github.com/nitrictech/nitric/cloud/common/deploy/image"
deploymentspb "github.com/nitrictech/nitric/core/pkg/proto/deployments/v1"
"github.com/pulumi/pulumi-docker/sdk/v4/go/docker"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/artifactregistry"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/cloudrunv2"
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/sql"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
)

type CloudBuild struct {
Expand All @@ -58,7 +54,7 @@ func (a *NitricGcpPulumiProvider) SqlDatabase(ctx *pulumi.Context, parent pulumi
}

if a.DatabaseMigrationBuild[name] == nil && config.GetImageUri() != "" {
clientContext := context.TODO()
// clientContext := context.TODO()

imageUriSplit := strings.Split(config.GetImageUri(), "/")
imageName := imageUriSplit[len(imageUriSplit)-1]
Expand All @@ -68,9 +64,9 @@ func (a *NitricGcpPulumiProvider) SqlDatabase(ctx *pulumi.Context, parent pulumi
return err
}

repo, err := artifactregistry.NewRepository(ctx, fmt.Sprintf("%s-migration-repo", name), &artifactregistry.RepositoryArgs{
repo, err := artifactregistry.NewRepository(ctx, fmt.Sprintf("%s-migration-repo-%s", a.StackId, name), &artifactregistry.RepositoryArgs{
Location: pulumi.String(a.Region),
RepositoryId: pulumi.Sprintf("%s-migration-repo", name),
RepositoryId: pulumi.Sprintf("%s-migration-repo-%s", a.StackId, name),
Format: pulumi.String("DOCKER"),
})
if err != nil {
Expand Down Expand Up @@ -99,111 +95,56 @@ func (a *NitricGcpPulumiProvider) SqlDatabase(ctx *pulumi.Context, parent pulumi

databaseUrl := pulumi.Sprintf("postgres://%s:%s@%s:%s/%s", "postgres", a.dbMasterPassword.Result, a.masterDb.PrivateIpAddress, "5432", name)

buildId := pulumi.All(databaseUrl, a.cloudBuildWorkerPool.ID().ToStringOutput(), image.Name).ApplyT(func(args []interface{}) (string, error) {
creds, err := google.FindDefaultCredentials(clientContext)
if err != nil {
return "", err
}

client, err := cloudbuild.NewClient(clientContext, option.WithCredentials(creds), option.WithQuotaProject(a.GcpConfig.ProjectId))
if err != nil {
return "", err
}

defer client.Close()

url := args[0].(string)
workerPoolId := args[1].(string)
imageUri := args[2].(string)

build, err := client.CreateBuild(clientContext, &cloudbuildpb.CreateBuildRequest{
Parent: fmt.Sprintf("projects/%s/locations/%s", a.GcpConfig.ProjectId, a.Region),
ProjectId: a.GcpConfig.ProjectId,
Build: &cloudbuildpb.Build{
Substitutions: map[string]string{
"_DATABASE_NAME": name,
"_DATABASE_URL": url,
// Run as google cloud run jobs instead of using cloud build
// This way we don't need to configre private worker pools (can share VPC config with cloud run services)

imageDigest := image.Sha256Digest.ApplyT(func(digest string) string {
// Generate the MD5 hash of the combined string
// TODO: Chances for collisions are low, but we should consider a better way to generate unique names
hash := md5.Sum([]byte(digest)) //#nosec G401 -- md5 used only to produce a unique ID from non-sensistive information (policy IDs)
md5Hash := hex.EncodeToString(hash[:])
// Truncate the MD5 hash to the first 63 characters if necessary
return md5Hash
}).(pulumi.StringOutput)

a.DatabaseMigrationBuild[name], err = cloudrunv2.NewJob(ctx, name+"-migration", &cloudrunv2.JobArgs{
Location: pulumi.String(a.Region),
StartExecutionToken: imageDigest,
DeletionProtection: pulumi.Bool(false),
Template: &cloudrunv2.JobTemplateArgs{
Template: &cloudrunv2.JobTemplateTemplateArgs{
VpcAccess: &cloudrunv2.JobTemplateTemplateVpcAccessArgs{
Connector: a.vpcConnector.SelfLink,
Egress: pulumi.String("PRIVATE_RANGES_ONLY"),
// TODO: Re-enable when pulumi network interface support is fixed for tear down
// NetworkInterfaces: cloudrunv2.JobTemplateTemplateVpcAccessNetworkInterfaceArray{
// &cloudrunv2.JobTemplateTemplateVpcAccessNetworkInterfaceArgs{
// Network: a.privateNetwork.ID(),
// Subnetwork: a.privateSubnet.ID(),
// },
// },
},
Steps: []*cloudbuildpb.BuildStep{
{
Name: imageUri,
Dir: "/",
Env: []string{
"NITRIC_DB_NAME=${_DATABASE_NAME}",
"DB_URL=${_DATABASE_URL}",
Containers: cloudrunv2.JobTemplateTemplateContainerArray{
&cloudrunv2.JobTemplateTemplateContainerArgs{
Image: image.Name,
Envs: cloudrunv2.JobTemplateTemplateContainerEnvArray{
&cloudrunv2.JobTemplateTemplateContainerEnvArgs{
Name: pulumi.String("NITRIC_DB_NAME"),
Value: pulumi.String(name),
},
&cloudrunv2.JobTemplateTemplateContainerEnvArgs{
Name: pulumi.String("DB_URL"),
Value: databaseUrl,
},
},
},
},
Options: &cloudbuildpb.BuildOptions{
Pool: &cloudbuildpb.BuildOptions_PoolOption{
Name: workerPoolId,
},
},
},
})
if err != nil {
return "", fmt.Errorf("error creating build for db %s: %w", name, err)
}

err = retry.Do(func() error {
metadata, err := build.Metadata()
if err != nil {
return retry.Unrecoverable(fmt.Errorf("failed to retrieve metadata: %w", err))
}

if metadata == nil {
return fmt.Errorf("unable to retrieve metadata")
}

currBuild, err := client.GetBuild(clientContext, &cloudbuildpb.GetBuildRequest{
Id: metadata.Build.Id,
Name: fmt.Sprintf("projects/%s/locations/%s/builds/%s", a.GcpConfig.ProjectId, a.Region, metadata.Build.Id),
ProjectId: a.GcpConfig.ProjectId,
})
if err != nil {
if strings.Contains(err.Error(), "not found") {
return fmt.Errorf("build operation not found: %w", err)
}
return fmt.Errorf("failed to poll build: %w", err)
}

if currBuild.Status == cloudbuildpb.Build_PENDING || currBuild.Status == cloudbuildpb.Build_WORKING || currBuild.Status == cloudbuildpb.Build_QUEUED {
return fmt.Errorf("build still in progress with status: %s", currBuild.Status)
}

if currBuild.Status == cloudbuildpb.Build_SUCCESS {
return nil
}

return retry.Unrecoverable(fmt.Errorf("build failed with status: %s", currBuild.Status))
}, retry.Attempts(10), retry.Delay(10*time.Second))
if err != nil {
return "", err
}

return build.Name(), nil
}).(pulumi.StringOutput)

res := &CloudBuild{
ID: buildId,
}

err = ctx.RegisterComponentResource("nitricgcp:cloudbuild:Build", name, res, pulumi.Parent(parent), pulumi.DependsOn([]pulumi.Resource{a.masterDb}))
if err != nil {
return err
}

res.ID = buildId

err = ctx.RegisterResourceOutputs(res, pulumi.Map{
"ID": buildId,
},
})
if err != nil {
return err
}

ctx.Export(fmt.Sprintf("migration-%s-build-id", name), buildId)
a.DatabaseMigrationBuild[name] = res
}

return nil
Expand Down
4 changes: 0 additions & 4 deletions cloud/gcp/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@ toolchain go1.23.0
require (
cloud.google.com/go/apigateway v1.7.0
cloud.google.com/go/batch v1.10.0
cloud.google.com/go/cloudbuild v1.17.0
cloud.google.com/go/cloudtasks v1.13.0
cloud.google.com/go/firestore v1.16.0
cloud.google.com/go/pubsub v1.42.0
cloud.google.com/go/secretmanager v1.14.0
cloud.google.com/go/storage v1.43.0
github.com/GoogleCloudPlatform/opentelemetry-operations-go/propagator v0.34.2
github.com/avast/retry-go v3.0.0+incompatible
github.com/aws/constructs-go/constructs/v10 v10.3.0
github.com/aws/jsii-runtime-go v1.99.0
github.com/cdktf/cdktf-provider-docker-go/docker/v11 v11.0.0
Expand All @@ -34,8 +32,6 @@ require (
github.com/onsi/gomega v1.34.2
github.com/pkg/errors v0.9.1
github.com/pulumi/pulumi-docker/sdk/v4 v4.1.0
github.com/pulumi/pulumi-gcp/sdk/v6 v6.67.1
github.com/pulumi/pulumi-std/sdk v1.6.2
github.com/pulumi/pulumi/sdk/v3 v3.133.0
github.com/samber/lo v1.38.1
github.com/uw-labs/lichen v0.1.7
Expand Down
Loading

0 comments on commit da6b0fc

Please sign in to comment.