Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(pubsub): add v2 topic and publishing samples #5107

Draft
wants to merge 2 commits into
base: pubsub-v2-samples-trunk
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 28 additions & 25 deletions pubsub/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,30 @@ module github.com/GoogleCloudPlatform/golang-samples/pubsub
go 1.21

require (
cloud.google.com/go/bigquery v1.62.0
cloud.google.com/go/iam v1.2.1
cloud.google.com/go/pubsub v1.44.0
cloud.google.com/go/bigquery v1.64.0
cloud.google.com/go/iam v1.2.2
cloud.google.com/go/pubsub v1.45.1
cloud.google.com/go/pubsub/v2 v2.0.0-00010101000000-000000000000
cloud.google.com/go/storage v1.43.0
cloud.google.com/go/trace v1.11.0
cloud.google.com/go/trace v1.11.2
github.com/GoogleCloudPlatform/golang-samples v0.0.0-20240820230436-761d0ae7aeff
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.24.1
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.13.0
github.com/googleapis/gax-go/v2 v2.14.0
github.com/linkedin/goavro/v2 v2.13.0
go.opentelemetry.io/otel v1.29.0
go.opentelemetry.io/otel/sdk v1.29.0
google.golang.org/api v0.197.0
google.golang.org/grpc v1.66.2
google.golang.org/protobuf v1.34.2
google.golang.org/api v0.211.0
google.golang.org/grpc v1.67.1
google.golang.org/protobuf v1.35.2
)

require (
cloud.google.com/go v0.115.1 // indirect
cloud.google.com/go/auth v0.9.3 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/auth v0.12.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand All @@ -48,18 +49,20 @@ require (
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/crypto v0.30.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.22.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/net v0.32.0 // indirect
golang.org/x/oauth2 v0.24.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect
)

replace cloud.google.com/go/pubsub/v2 => /Users/hongalex/code/cloud/google-cloud-go/pubsub/v2
120 changes: 60 additions & 60 deletions pubsub/go.sum

Large diffs are not rendered by default.

44 changes: 29 additions & 15 deletions pubsub/topics/add_users.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ package topics
import (
"context"
"fmt"
"io"

"cloud.google.com/go/iam"
"cloud.google.com/go/pubsub"
"cloud.google.com/go/iam/apiv1/iampb"
"cloud.google.com/go/pubsub/v2"
)

func addUsers(projectID, topicID string) error {
func addUsers(w io.Writer, projectID, topicID string) error {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low

Per the Google Go style guide, function names should be descriptive and in camelCase. Consider renaming this function to addUsersToTopic.

Suggested change
func addUsers(w io.Writer, projectID, topicID string) error {
func addUsersToTopic(w io.Writer, projectID, topicID string) error {

// projectID := "my-project-id"
// topicID := "my-topic"
ctx := context.Background()
Expand All @@ -33,21 +34,34 @@ func addUsers(projectID, topicID string) error {
}
defer client.Close()

topic := client.Topic(topicID)
policy, err := topic.IAM().Policy(ctx)
topicName := fmt.Sprintf("projects/%s/topics/%s", projectID, topicID)
req := &iampb.GetIamPolicyRequest{
Resource: topicName,
}
policy, err := client.TopicAdminClient.GetIamPolicy(ctx, req)
if err != nil {
return fmt.Errorf("Policy: %w", err)
return fmt.Errorf("error calling GetIamPolicy: %w", err)
}
b1 := &iampb.Binding{
Role: "roles/viewer",
Members: []string{"allUsers"},
}
b2 := &iampb.Binding{
Role: "roles/editor",
// Other valid prefixes are "serviceAccount:", "user:"
// See the documentation for more values.
Members: []string{"group:[email protected]"},
}
// Other valid prefixes are "serviceAccount:", "user:"
// See the documentation for more values.
policy.Add(iam.AllUsers, iam.Viewer)
policy.Add("group:[email protected]", iam.Editor)
if err := topic.IAM().SetPolicy(ctx, policy); err != nil {
return fmt.Errorf("SetPolicy: %w", err)
policy.Bindings = append(policy.Bindings, b1, b2)

setRequest := &iampb.SetIamPolicyRequest{
Resource: topicName,
Policy: policy,
}
_, err = client.TopicAdminClient.SetIamPolicy(ctx, setRequest)
if err != nil {
return fmt.Errorf("error calling SetIamPolicy: %w", err)
}
// NOTE: It may be necessary to retry this operation if IAM policies are
// being modified concurrently. SetPolicy will return an error if the policy
// was modified since it was retrieved.
return nil
}

Expand Down
8 changes: 6 additions & 2 deletions pubsub/topics/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
"fmt"
"io"

"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
)

func create(w io.Writer, projectID, topicID string) error {
Expand All @@ -33,7 +34,10 @@ func create(w io.Writer, projectID, topicID string) error {
}
defer client.Close()

t, err := client.CreateTopic(ctx, topicID)
topic := &pubsubpb.Topic{
Name: fmt.Sprintf("projects/%s/topics/%s", projectID, topicID),
}
t, err := client.TopicAdminClient.CreateTopic(ctx, topic)
if err != nil {
return fmt.Errorf("CreateTopic: %w", err)
}
Expand Down
32 changes: 20 additions & 12 deletions pubsub/topics/create_topic_gcs_ingestion.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,18 @@ import (
"io"
"time"

"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
"google.golang.org/protobuf/types/known/timestamppb"
)

func createTopicWithCloudStorageIngestion(w io.Writer, projectID, topicID, bucket, matchGlob, minimumObjectCreateTime string) error {
func createTopicWithCloudStorageIngestion(w io.Writer, projectID, topicID, bucket, matchGlob, minimumObjectCreateTime, delimiter string) error {
// projectID := "my-project-id"
// topicID := "my-topic"
// bucket := "my-bucket"
// matchGlob := "**.txt"
// minimumObjectCreateTime := "2006-01-02T15:04:05Z"
// delimiter := ","

ctx := context.Background()
client, err := pubsub.NewClient(ctx, projectID)
Expand All @@ -43,20 +46,25 @@ func createTopicWithCloudStorageIngestion(w io.Writer, projectID, topicID, bucke
return err
}

cfg := &pubsub.TopicConfig{
IngestionDataSourceSettings: &pubsub.IngestionDataSourceSettings{
Source: &pubsub.IngestionDataSourceCloudStorage{
Bucket: bucket,
// Alternatively, can be Avro or PubSubAvro formats. See
InputFormat: &pubsub.IngestionDataSourceCloudStorageTextFormat{
Delimiter: ",",
topicpb := &pubsubpb.Topic{
Name: fmt.Sprintf("projects/%s/topics/%s", projectID, topicID),
IngestionDataSourceSettings: &pubsubpb.IngestionDataSourceSettings{
Source: &pubsubpb.IngestionDataSourceSettings_CloudStorage_{
CloudStorage: &pubsubpb.IngestionDataSourceSettings_CloudStorage{
Bucket: bucket,
// Alternatively, can be Avro or PubSubAvro formats. See
InputFormat: &pubsubpb.IngestionDataSourceSettings_CloudStorage_TextFormat_{
TextFormat: &pubsubpb.IngestionDataSourceSettings_CloudStorage_TextFormat{
Delimiter: &delimiter,
},
},
MatchGlob: matchGlob,
MinimumObjectCreateTime: timestamppb.New(minCreateTime),
},
MatchGlob: matchGlob,
MinimumObjectCreateTime: minCreateTime,
},
},
}
t, err := client.CreateTopicWithConfig(ctx, topicID, cfg)
t, err := client.TopicAdminClient.CreateTopic(ctx, topicpb)
if err != nil {
return fmt.Errorf("CreateTopic: %w", err)
}
Expand Down
25 changes: 14 additions & 11 deletions pubsub/topics/create_topic_kinesis_ingestion.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
"fmt"
"io"

"cloud.google.com/go/pubsub"
pubsub "cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
)

func createTopicWithKinesisIngestion(w io.Writer, projectID, topicID string) error {
Expand All @@ -38,21 +39,23 @@ func createTopicWithKinesisIngestion(w io.Writer, projectID, topicID string) err
}
defer client.Close()

cfg := &pubsub.TopicConfig{
IngestionDataSourceSettings: &pubsub.IngestionDataSourceSettings{
Source: &pubsub.IngestionDataSourceAWSKinesis{
StreamARN: streamARN,
ConsumerARN: consumerARN,
AWSRoleARN: awsRoleARN,
GCPServiceAccount: gcpServiceAccount,
topicpb := &pubsubpb.Topic{
IngestionDataSourceSettings: &pubsubpb.IngestionDataSourceSettings{
Source: &pubsubpb.IngestionDataSourceSettings_AwsKinesis_{
AwsKinesis: &pubsubpb.IngestionDataSourceSettings_AwsKinesis{
StreamArn: streamARN,
ConsumerArn: consumerARN,
AwsRoleArn: awsRoleARN,
GcpServiceAccount: gcpServiceAccount,
},
},
},
}
t, err := client.CreateTopicWithConfig(ctx, topicID, cfg)
topicpb, err = client.TopicAdminClient.CreateTopic(ctx, topicpb)
if err != nil {
return fmt.Errorf("CreateTopic: %w", err)
return fmt.Errorf("failed to create topic with kinesis: %w", err)
}
fmt.Fprintf(w, "Kinesis topic created: %v\n", t)
fmt.Fprintf(w, "Kinesis topic created: %v\n", topicpb)
return nil
}

Expand Down
14 changes: 9 additions & 5 deletions pubsub/topics/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ import (
"fmt"
"io"

"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
)

func delete(w io.Writer, projectID, topicID string) error {
Expand All @@ -33,11 +34,14 @@ func delete(w io.Writer, projectID, topicID string) error {
}
defer client.Close()

t := client.Topic(topicID)
if err := t.Delete(ctx); err != nil {
return fmt.Errorf("Delete: %w", err)
req := &pubsubpb.DeleteTopicRequest{
Topic: fmt.Sprintf("projects/%s/topics/%s", projectID, topicID),
}
fmt.Fprintf(w, "Deleted topic: %v\n", t)
err = client.TopicAdminClient.DeleteTopic(ctx, req)
if err != nil {
return fmt.Errorf("failed to delete topic: %w", err)
}
fmt.Fprintln(w, "Deleted topic")
return nil
}

Expand Down
21 changes: 12 additions & 9 deletions pubsub/topics/list_subs.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,38 @@ package topics
import (
"context"
"fmt"
"io"

"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
"google.golang.org/api/iterator"
)

func listSubscriptions(projectID, topicID string) ([]*pubsub.Subscription, error) {
func listSubscriptions(w io.Writer, projectID, topicID string) error {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

low

Per the Google Go style guide, function names should be descriptive and in camelCase. Consider renaming this function to listTopicSubscriptions.

Suggested change
func listSubscriptions(w io.Writer, projectID, topicID string) error {
func listTopicSubscriptions(w io.Writer, projectID, topicID string) error {

// projectID := "my-project-id"
// topicName := "projects/sample-248520/topics/ocr-go-test-topic"
ctx := context.Background()
client, err := pubsub.NewClient(ctx, projectID)
if err != nil {
return nil, fmt.Errorf("pubsub.NewClient: %w", err)
return fmt.Errorf("pubsub.NewClient: %w", err)
}
defer client.Close()

var subs []*pubsub.Subscription

it := client.Topic(topicID).Subscriptions(ctx)
req := &pubsubpb.ListTopicSubscriptionsRequest{
Topic: fmt.Sprintf("projects/%s/topics/%s", projectID, topicID),
}
it := client.TopicAdminClient.ListTopicSubscriptions(ctx, req)
for {
sub, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return nil, fmt.Errorf("Next: %w", err)
return fmt.Errorf("error listing topic subscriptions: %w", err)
}
subs = append(subs, sub)
fmt.Fprintf(w, "got subscription: %s\n", sub)
}
return subs, nil
return nil
}

// [END pubsub_list_topic_subscriptions]
23 changes: 13 additions & 10 deletions pubsub/topics/list_topics.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,35 +18,38 @@ package topics
import (
"context"
"fmt"
"io"

"cloud.google.com/go/pubsub"
"cloud.google.com/go/pubsub/v2"
"cloud.google.com/go/pubsub/v2/apiv1/pubsubpb"
"google.golang.org/api/iterator"
)

func list(projectID string) ([]*pubsub.Topic, error) {
func listTopics(w io.Writer, projectID string) error {
// projectID := "my-project-id"
ctx := context.Background()
client, err := pubsub.NewClient(ctx, projectID)
if err != nil {
return nil, fmt.Errorf("pubsub.NewClient: %w", err)
return fmt.Errorf("pubsub.NewClient: %w", err)
}
defer client.Close()

var topics []*pubsub.Topic

it := client.Topics(ctx)
req := &pubsubpb.ListTopicsRequest{
Project: fmt.Sprintf("projects/%s", projectID),
}
it := client.TopicAdminClient.ListTopics(ctx, req)
for {
topic, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
return nil, fmt.Errorf("Next: %w", err)
return fmt.Errorf("error listing topics: %w", err)
}
topics = append(topics, topic)
}
fmt.Fprintf(w, "got topic: %s\n", topic)

return topics, nil
}
return nil
}

// [END pubsub_list_topics]
Loading