From 7845b38f2b2e613fd85ea3da8fa614045047ac2b Mon Sep 17 00:00:00 2001 From: wayblink Date: Wed, 10 Jan 2024 19:03:20 +0800 Subject: [PATCH] Refine proto and swag (#281) Signed-off-by: wayblink --- build.sh | 4 +-- core/backup_server.go | 12 ++++---- core/proto/backup.proto | 10 +++---- core/proto/backuppb/backup.pb.go | 34 ++++++++++----------- docs/docs.go | 11 ++++--- docs/swagger.json | 11 ++++--- docs/swagger.yaml | 10 +++---- proto_gen_go.sh => gen_proto.sh | 0 gen_swag.sh | 51 ++++++++++++++++++++++++++++++++ 9 files changed, 96 insertions(+), 47 deletions(-) rename proto_gen_go.sh => gen_proto.sh (100%) create mode 100755 gen_swag.sh diff --git a/build.sh b/build.sh index 7b15aa0b..bb5313a9 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,4 @@ go get +./gen_proto.sh +./gen_swag.sh go build -./proto_gen_go.sh -swag init diff --git a/core/backup_server.go b/core/backup_server.go index 9bda060a..db9abf88 100644 --- a/core/backup_server.go +++ b/core/backup_server.go @@ -132,7 +132,7 @@ func wrapHandler(handle handlerFunc) gin.HandlerFunc { // @Tags Backup // @Accept application/json // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @Param object body backuppb.CreateBackupRequest true "CreateBackupRequest JSON" // @Success 200 {object} backuppb.BackupInfoResponse // @Router /create [post] @@ -156,7 +156,7 @@ func (h *Handlers) handleCreateBackup(c *gin.Context) (interface{}, error) { // @Description List all backups in current storage // @Tags Backup // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @Param collection_name query string true "collection_name" // @Success 200 {object} backuppb.ListBackupsResponse // @Router /list [get] @@ -178,7 +178,7 @@ func (h *Handlers) handleListBackups(c *gin.Context) (interface{}, error) { // @Description Get the backup with the given name or id // @Tags Backup // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @Param backup_name query string true "backup_name" // @Param backup_id query string true "backup_id" // @Success 200 {object} backuppb.BackupInfoResponse @@ -202,7 +202,7 @@ func (h *Handlers) handleGetBackup(c *gin.Context) (interface{}, error) { // @Description Delete a backup with the given name // @Tags Backup // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @Param backup_name query string true "backup_name" // @Success 200 {object} backuppb.DeleteBackupResponse // @Router /delete [delete] @@ -222,7 +222,7 @@ func (h *Handlers) handleDeleteBackup(c *gin.Context) (interface{}, error) { // @Tags Restore // @Accept application/json // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @Param object body backuppb.RestoreBackupRequest true "RestoreBackupRequest JSON" // @Success 200 {object} backuppb.RestoreBackupResponse // @Router /restore [post] @@ -251,7 +251,7 @@ func (h *Handlers) handleRestoreBackup(c *gin.Context) (interface{}, error) { // @Description Get restore task state with the given id // @Tags Restore // @Produce application/json -// @Param request_id header string true "request_id" +// @Param request_id header string false "request_id" // @param id query string true "id" // @Success 200 {object} backuppb.RestoreBackupResponse // @Router /get_restore [get] diff --git a/core/proto/backup.proto b/core/proto/backup.proto index 107d98af..9f063f91 100644 --- a/core/proto/backup.proto +++ b/core/proto/backup.proto @@ -1,7 +1,7 @@ syntax = "proto3"; package milvus.proto.backup; -import "google/protobuf/struct.proto"; +import "google/protobuf/struct.proto"; // proto define option go_package="github.com/zilliztech/milvus-backup/core/proto/backuppb"; @@ -253,17 +253,17 @@ message RestoreBackupRequest { string path = 8; // database and collections to restore. A json string. To support database. 2023.7.7 google.protobuf.Value db_collections = 9; - // if true only restore meta + // if true only restore meta, not restore data bool metaOnly = 10; // if true restore index info bool restoreIndex = 11; // if true use autoindex when restore vector index bool useAutoIndex = 12; - // if true drop the collections + // if true, drop existing target collection before create bool dropExistCollection = 13; - // if true drop index info + // if true, drop existing index of target collection before create bool dropExistIndex = 14; - // if true will skip create collections + // if true, will skip collection, use when collection exist, restore index or data bool skipCreateCollection = 15; } diff --git a/core/proto/backuppb/backup.pb.go b/core/proto/backuppb/backup.pb.go index b2c9c01c..7e5efd1e 100644 --- a/core/proto/backuppb/backup.pb.go +++ b/core/proto/backuppb/backup.pb.go @@ -339,8 +339,8 @@ type CollectionBackupInfo struct { PartitionBackups []*PartitionBackupInfo `protobuf:"bytes,13,rep,name=partition_backups,json=partitionBackups,proto3" json:"partition_backups,omitempty"` // logical time of backup, used for restore BackupTimestamp uint64 `protobuf:"varint,14,opt,name=backup_timestamp,json=backupTimestamp,proto3" json:"backup_timestamp,omitempty"` - Size int64 `protobuf:"varint,15,opt,name=size,proto3" json:"size"` - HasIndex bool `protobuf:"varint,16,opt,name=has_index,json=hasIndex,proto3" json:"has_index"` + Size int64 `protobuf:"varint,15,opt,name=size,proto3" json:"size,omitempty"` + HasIndex bool `protobuf:"varint,16,opt,name=has_index,json=hasIndex,proto3" json:"has_index,omitempty"` IndexInfos []*IndexInfo `protobuf:"bytes,17,rep,name=index_infos,json=indexInfos,proto3" json:"index_infos,omitempty"` LoadState string `protobuf:"bytes,18,opt,name=load_state,json=loadState,proto3" json:"load_state,omitempty"` // physical unix time of backup @@ -514,7 +514,7 @@ type PartitionBackupInfo struct { CollectionId int64 `protobuf:"varint,3,opt,name=collection_id,json=collectionId,proto3" json:"collection_id,omitempty"` // array of segment backup SegmentBackups []*SegmentBackupInfo `protobuf:"bytes,4,rep,name=segment_backups,json=segmentBackups,proto3" json:"segment_backups,omitempty"` - Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size"` + Size int64 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` LoadState string `protobuf:"bytes,6,opt,name=load_state,json=loadState,proto3" json:"load_state,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -598,7 +598,7 @@ type SegmentBackupInfo struct { Binlogs []*FieldBinlog `protobuf:"bytes,5,rep,name=binlogs,proto3" json:"binlogs,omitempty"` Statslogs []*FieldBinlog `protobuf:"bytes,6,rep,name=statslogs,proto3" json:"statslogs,omitempty"` Deltalogs []*FieldBinlog `protobuf:"bytes,7,rep,name=deltalogs,proto3" json:"deltalogs,omitempty"` - Size int64 `protobuf:"varint,8,opt,name=size,proto3" json:"size"` + Size int64 `protobuf:"varint,8,opt,name=size,proto3" json:"size,omitempty"` // separate segments into multi groups by size, // segments in one group will be copied into one directory during backup // and will bulkinsert in one call during restore @@ -710,7 +710,7 @@ type BackupInfo struct { BackupTimestamp uint64 `protobuf:"varint,8,opt,name=backup_timestamp,json=backupTimestamp,proto3" json:"backup_timestamp,omitempty"` // array of collection backup CollectionBackups []*CollectionBackupInfo `protobuf:"bytes,9,rep,name=collection_backups,json=collectionBackups,proto3" json:"collection_backups,omitempty"` - Size int64 `protobuf:"varint,10,opt,name=size,proto3" json:"size"` + Size int64 `protobuf:"varint,10,opt,name=size,proto3" json:"size,omitempty"` MilvusVersion string `protobuf:"bytes,11,opt,name=milvus_version,json=milvusVersion,proto3" json:"milvus_version,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1048,7 +1048,7 @@ type BackupInfoResponse struct { // error msg if fail Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"` // backup info entity - Data *BackupInfo `protobuf:"bytes,4,opt,name=data,proto3" json:"data"` + Data *BackupInfo `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1249,7 +1249,7 @@ type ListBackupsResponse struct { // error msg if fail Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"` // backup info entities - Data []*BackupInfo `protobuf:"bytes,4,rep,name=data,proto3" json:"data"` + Data []*BackupInfo `protobuf:"bytes,4,rep,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1436,17 +1436,17 @@ type RestoreBackupRequest struct { Path string `protobuf:"bytes,8,opt,name=path,proto3" json:"path,omitempty"` // database and collections to restore. A json string. To support database. 2023.7.7 DbCollections *_struct.Value `protobuf:"bytes,9,opt,name=db_collections,json=dbCollections,proto3" json:"db_collections,omitempty"` - // if true only restore meta + // if true only restore meta, not restore data MetaOnly bool `protobuf:"varint,10,opt,name=metaOnly,proto3" json:"metaOnly,omitempty"` // if true restore index info RestoreIndex bool `protobuf:"varint,11,opt,name=restoreIndex,proto3" json:"restoreIndex,omitempty"` // if true use autoindex when restore vector index UseAutoIndex bool `protobuf:"varint,12,opt,name=useAutoIndex,proto3" json:"useAutoIndex,omitempty"` - // if true drop the collections + // if true, drop existing target collection before create DropExistCollection bool `protobuf:"varint,13,opt,name=dropExistCollection,proto3" json:"dropExistCollection,omitempty"` - // if true drop index info + // if true, drop existing index of target collection before create DropExistIndex bool `protobuf:"varint,14,opt,name=dropExistIndex,proto3" json:"dropExistIndex,omitempty"` - // if true will skip create collections + // if true, will skip collection, use when collection exist, restore index or data SkipCreateCollection bool `protobuf:"varint,15,opt,name=skipCreateCollection,proto3" json:"skipCreateCollection,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1679,8 +1679,8 @@ type RestoreCollectionTask struct { CollBackup *CollectionBackupInfo `protobuf:"bytes,6,opt,name=coll_backup,json=collBackup,proto3" json:"coll_backup,omitempty"` TargetCollectionName string `protobuf:"bytes,7,opt,name=target_collection_name,json=targetCollectionName,proto3" json:"target_collection_name,omitempty"` PartitionRestoreTasks []*RestorePartitionTask `protobuf:"bytes,8,rep,name=partition_restore_tasks,json=partitionRestoreTasks,proto3" json:"partition_restore_tasks,omitempty"` - RestoredSize int64 `protobuf:"varint,9,opt,name=restored_size,json=restoredSize,proto3" json:"restored_size"` - ToRestoreSize int64 `protobuf:"varint,10,opt,name=to_restore_size,json=toRestoreSize,proto3" json:"to_restore_size"` + RestoredSize int64 `protobuf:"varint,9,opt,name=restored_size,json=restoredSize,proto3" json:"restored_size,omitempty"` + ToRestoreSize int64 `protobuf:"varint,10,opt,name=to_restore_size,json=toRestoreSize,proto3" json:"to_restore_size,omitempty"` Progress int32 `protobuf:"varint,11,opt,name=progress,proto3" json:"progress,omitempty"` TargetDbName string `protobuf:"bytes,12,opt,name=target_db_name,json=targetDbName,proto3" json:"target_db_name,omitempty"` // if true only restore meta @@ -1858,8 +1858,8 @@ type RestoreBackupTask struct { StartTime int64 `protobuf:"varint,4,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` EndTime int64 `protobuf:"varint,5,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` CollectionRestoreTasks []*RestoreCollectionTask `protobuf:"bytes,6,rep,name=collection_restore_tasks,json=collectionRestoreTasks,proto3" json:"collection_restore_tasks,omitempty"` - RestoredSize int64 `protobuf:"varint,7,opt,name=restored_size,json=restoredSize,proto3" json:"restored_size"` - ToRestoreSize int64 `protobuf:"varint,8,opt,name=to_restore_size,json=toRestoreSize,proto3" json:"to_restore_size"` + RestoredSize int64 `protobuf:"varint,7,opt,name=restored_size,json=restoredSize,proto3" json:"restored_size,omitempty"` + ToRestoreSize int64 `protobuf:"varint,8,opt,name=to_restore_size,json=toRestoreSize,proto3" json:"to_restore_size,omitempty"` Progress int32 `protobuf:"varint,9,opt,name=progress,proto3" json:"progress,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1962,7 +1962,7 @@ type RestoreBackupResponse struct { // error msg if fail Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"` // restore task info entity - Data *RestoreBackupTask `protobuf:"bytes,4,opt,name=data,proto3" json:"data"` + Data *RestoreBackupTask `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2123,7 +2123,7 @@ type Binlog struct { TimestampFrom uint64 `protobuf:"varint,2,opt,name=timestamp_from,json=timestampFrom,proto3" json:"timestamp_from,omitempty"` TimestampTo uint64 `protobuf:"varint,3,opt,name=timestamp_to,json=timestampTo,proto3" json:"timestamp_to,omitempty"` LogPath string `protobuf:"bytes,4,opt,name=log_path,json=logPath,proto3" json:"log_path,omitempty"` - LogSize int64 `protobuf:"varint,5,opt,name=log_size,json=logSize,proto3" json:"log_size"` + LogSize int64 `protobuf:"varint,5,opt,name=log_size,json=logSize,proto3" json:"log_size,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` diff --git a/docs/docs.go b/docs/docs.go index f0f0a139..cd2eb4ff 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -233,8 +233,7 @@ const docTemplate = `{ "type": "string", "description": "request_id", "name": "request_id", - "in": "header", - "required": true + "in": "header" }, { "description": "RestoreBackupRequest JSON", @@ -791,15 +790,15 @@ const docTemplate = `{ "type": "string" }, "dropExistCollection": { - "description": "if true drop the collections", + "description": "if true, drop existing target collection before create", "type": "boolean" }, "dropExistIndex": { - "description": "if true drop index info", + "description": "if true, drop existing index of target collection before create", "type": "boolean" }, "metaOnly": { - "description": "if true only restore meta", + "description": "if true only restore meta, not restore data", "type": "boolean" }, "path": { @@ -815,7 +814,7 @@ const docTemplate = `{ "type": "boolean" }, "skipCreateCollection": { - "description": "if true will skip create collections", + "description": "if true, will skip collection, use when collection exist, restore index or data", "type": "boolean" }, "useAutoIndex": { diff --git a/docs/swagger.json b/docs/swagger.json index 96c7f454..c11e85c5 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -225,8 +225,7 @@ "type": "string", "description": "request_id", "name": "request_id", - "in": "header", - "required": true + "in": "header" }, { "description": "RestoreBackupRequest JSON", @@ -783,15 +782,15 @@ "type": "string" }, "dropExistCollection": { - "description": "if true drop the collections", + "description": "if true, drop existing target collection before create", "type": "boolean" }, "dropExistIndex": { - "description": "if true drop index info", + "description": "if true, drop existing index of target collection before create", "type": "boolean" }, "metaOnly": { - "description": "if true only restore meta", + "description": "if true only restore meta, not restore data", "type": "boolean" }, "path": { @@ -807,7 +806,7 @@ "type": "boolean" }, "skipCreateCollection": { - "description": "if true will skip create collections", + "description": "if true, will skip collection, use when collection exist, restore index or data", "type": "boolean" }, "useAutoIndex": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 15559124..fbab5399 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -382,13 +382,13 @@ definitions: database. 2023.7.7 type: string dropExistCollection: - description: if true drop the collections + description: if true, drop existing target collection before create type: boolean dropExistIndex: - description: if true drop index info + description: if true, drop existing index of target collection before create type: boolean metaOnly: - description: if true only restore meta + description: if true only restore meta, not restore data type: boolean path: description: if bucket_name and path is set. will override bucket/path in @@ -401,7 +401,8 @@ definitions: description: if true restore index info type: boolean skipCreateCollection: - description: if true will skip create collections + description: if true, will skip collection, use when collection exist, restore + index or data type: boolean useAutoIndex: description: if true use autoindex when restore vector index @@ -709,7 +710,6 @@ paths: - description: request_id in: header name: request_id - required: true type: string - description: RestoreBackupRequest JSON in: body diff --git a/proto_gen_go.sh b/gen_proto.sh similarity index 100% rename from proto_gen_go.sh rename to gen_proto.sh diff --git a/gen_swag.sh b/gen_swag.sh new file mode 100755 index 00000000..16a7d113 --- /dev/null +++ b/gen_swag.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +PWD := $(shell pwd) + +PROGRAM=${PWD} +GOPATH=$(go env GOPATH) +BACK_PROTO_DIR=$PROGRAM/core/proto/ +GOOGLE_PROTO_DIR=${PROGRAM}/build/thirdparty/protobuf-src/src/ + +echo ${PROGRAM} +export protoc=${PROGRAM}/build/thirdparty/protobuf-build/protoc +echo `${protoc} --version` +which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/protoc-gen-go@v1.3.2) + +if [ -z $GOPATH ]; then + printf "Error: the environment variable GOPATH is not set, please set it before running %s\n" $PROGRAM > /dev/stderr + exit 1 +fi + +export PATH=${GOPATH}/bin:$PATH +echo `which protoc-gen-go` +echo ${BACK_PROTO_DIR} + +pushd ${BACK_PROTO_DIR} + +mkdir -p backuppb + +sed -i "" -e "s/google.protobuf.Value db_collections/string db_collections/g" backup.proto +sed -i "" -e "s/import \"google\/protobuf\/struct.proto\"; \/\/ proto define/\/\/ proto define/g" backup.proto + +${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./backuppb backup.proto + +# remove has_index omitempty +sed -i "" -e "s/has_index,omitempty/has_index/g" ./backuppb/backup.pb.go +# remove data omitempty +sed -i "" -e "s/data,omitempty/data/g" ./backuppb/backup.pb.go +# remove size omitempty +sed -i "" -e "s/size,omitempty/size/g" ./backuppb/backup.pb.go + +popd + +swag init + +pushd ${BACK_PROTO_DIR} + +sed -i "" -e "s/string db_collections/google.protobuf.Value db_collections/g" backup.proto +sed -i "" -e "s/\/\/ proto define/import \"google\/protobuf\/struct.proto\"; \/\/ proto define/g" backup.proto + +${protoc} --proto_path="${GOOGLE_PROTO_DIR}" --proto_path=. --go_out=plugins=grpc,paths=source_relative:./backuppb backup.proto + +popd \ No newline at end of file