-
-
Notifications
You must be signed in to change notification settings - Fork 59
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add to support single table with flexible support for multiple partition keys. #181
base: main
Are you sure you want to change the base?
Changes from 4 commits
7d22b76
165e3d6
6fd67a8
c5a8a24
b1745bd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
package sharding | ||
|
||
import ( | ||
"context" | ||
"errors" | ||
"fmt" | ||
"hash/crc32" | ||
|
@@ -15,15 +16,20 @@ import ( | |
) | ||
|
||
var ( | ||
ErrMissingShardingKey = errors.New("sharding key or id required, and use operator =") | ||
ErrInvalidID = errors.New("invalid id format") | ||
ErrInsertDiffSuffix = errors.New("can not insert different suffix table in one query ") | ||
ErrMissingShardingKey = errors.New("sharding key or id required, and use operator =") | ||
ErrInvalidID = errors.New("invalid id format") | ||
ErrInsertDiffSuffix = errors.New("can not insert different suffix table in one query ") | ||
ErrShardingKeyNotExistInContext = errors.New("the value passed in the context is not the sharding key") | ||
ErrMissingTableName = errors.New("table name is required") | ||
) | ||
|
||
var ( | ||
ShardingIgnoreStoreKey = "sharding_ignore" | ||
) | ||
|
||
// ContextKeyForShardingKey is the context key for sharding key. | ||
const ContextKeyForShardingKey = "sharding_key" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 改成 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 好的~ |
||
|
||
type Sharding struct { | ||
*gorm.DB | ||
ConnPool *ConnPool | ||
|
@@ -46,6 +52,10 @@ type Config struct { | |
// For example, for a product order table, you may want to split the rows by `user_id`. | ||
ShardingKey string | ||
|
||
// logical table name.Suport multiple table names with same sharding key. | ||
// For example, for user and order table, you may want to shard by `user_id`. | ||
TableNames []string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个是必须要的吗?我看上一次都没有这个,这样会增加 config 理解的复杂度,容易产生使用上的歧义。 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 我准备要合并这个功能,但你得确保尽量少的改动 API,非必要的时候不要引入新的 API 设计。 TableNames 这个我感觉无法接受,这会让整个 Sharding 对于 table 的组织配置变得复杂。最初的设计进需要考虑 sharding_key,而 table 本身是由 Gorm 的 因此这里不应该存在特别配置 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
|
||
// NumberOfShards specifies how many tables you want to sharding. | ||
NumberOfShards uint | ||
|
||
|
@@ -110,6 +120,56 @@ func Register(config Config, tables ...any) *Sharding { | |
} | ||
} | ||
|
||
// enables sharding for a single table with flexible support for multiple partition keys. | ||
func RegisterWithKeys(configs []Config) (*Sharding, error) { | ||
mapConfig := make(map[string]Config, len(configs)) | ||
for _, config := range configs { | ||
for _, tableName := range config.TableNames { | ||
configKey, err := generateConfigsKey(tableName, config.ShardingKey) | ||
if err != nil { | ||
return nil, err | ||
} | ||
mapConfig[configKey] = config | ||
} | ||
} | ||
return &Sharding{ | ||
configs: mapConfig, | ||
}, nil | ||
} | ||
|
||
// generates the key for the sharding config. | ||
func generateConfigsKey(tableName, shardingKey string) (string, error) { | ||
// Table name cannot be empty | ||
if tableName == "" { | ||
return "", ErrMissingTableName | ||
} | ||
if shardingKey == "" { | ||
return "", ErrMissingShardingKey | ||
} | ||
return fmt.Sprintf("%s_%s", tableName, shardingKey), nil | ||
} | ||
|
||
// get the configs key for using it to get the sharding config. | ||
func (s *Sharding) getConfigKey(ctx context.Context, tableName string) (string, error) { | ||
configKey := tableName | ||
if shardingKey, ok := ctx.Value(ContextKeyForShardingKey).(string); ok { | ||
// If sharding key is set in context, use it to get the sharding config. | ||
configKey = fmt.Sprintf("%s_%s", tableName, shardingKey) | ||
} else { | ||
// If sharding key is not set in context, use the table name as the key. | ||
return configKey, nil | ||
} | ||
|
||
// check if the sharding key exists in the configs. | ||
_, exis := s.configs[configKey] | ||
if !exis { | ||
return "", ErrShardingKeyNotExistInContext | ||
} | ||
|
||
// If sharding key is not set in context, use the table name as the key. | ||
return configKey, nil | ||
} | ||
|
||
func (s *Sharding) compile() error { | ||
if s.configs == nil { | ||
s.configs = make(map[string]Config) | ||
|
@@ -297,7 +357,7 @@ func (s *Sharding) switchConn(db *gorm.DB) { | |
} | ||
|
||
// resolve split the old query to full table query and sharding table query | ||
func (s *Sharding) resolve(query string, args ...any) (ftQuery, stQuery, tableName string, err error) { | ||
func (s *Sharding) resolve(ctx context.Context, query string, args ...any) (ftQuery, stQuery, tableName string, err error) { | ||
ftQuery = query | ||
stQuery = query | ||
if len(s.configs) == 0 { | ||
|
@@ -344,7 +404,12 @@ func (s *Sharding) resolve(query string, args ...any) (ftQuery, stQuery, tableNa | |
} | ||
|
||
tableName = table.Name.Name | ||
r, ok := s.configs[tableName] | ||
key := tableName | ||
key, err = s.getConfigKey(ctx, tableName) | ||
if err != nil { | ||
return | ||
} | ||
r, ok := s.configs[key] | ||
if !ok { | ||
return | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
看起来这个思路是可行,如果能完善,我觉得我们可以引入这个改进。
但
sharding_key
和%s_%v
这种在实际应用的时候需要掌握这套规则,不是一个完整的 API 设计。例如
sharding_key
应该是一个const
,而key
的我没想到好的解决方法,目前这样,以后其他人使用会存在各种各样的问题,他们很容易遇到无法命中配置。如果是
sharding_key
已经传了,但sharding.configs
又找不到对应配置的情况下,应该直接报错。There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
有道理,这两点我完善一下~