Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add update subpath #107

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Add update subpath #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7a202b2
Adding subpaths to Object Queries
udnay Feb 22, 2025
c19817a
Adding subpaths to client functions
udnay Feb 24, 2025
e8ff2d3
Add rebuild test
udnay Feb 24, 2025
0d723f7
Add update test
udnay Feb 24, 2025
41be686
Add combined rebuild/upload test
udnay Feb 24, 2025
c792941
Move launch and tasks to an example
udnay Feb 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmd/fuzz-test/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ func runIteration(ctx context.Context, client *dlc.Client, project int64, operat
return -1, fmt.Errorf("failed to apply operation %s: %w", operation.String(), err)
}

version, _, err := client.Update(ctx, project, dirs.Base(project))
version, _, err := client.Update(ctx, project, dirs.Base(project), nil)
if err != nil {
return -1, fmt.Errorf("failed to update project %d: %w", project, err)
}
Expand All @@ -414,12 +414,12 @@ func runIteration(ctx context.Context, client *dlc.Client, project int64, operat
return -1, fmt.Errorf("failed to create reset dir %s: %w", dirs.Reset(project), err)
}

_, err = client.Rebuild(ctx, project, "", nil, dirs.Reset(project), nil, "", nil, false)
_, err = client.Rebuild(ctx, project, "", nil, dirs.Reset(project), nil, nil, "", nil, false)
if err != nil {
return -1, fmt.Errorf("failed to rebuild reset project %d: %w", project, err)
}

_, err = client.Rebuild(ctx, project, "", nil, dirs.OneStep(project), nil, "", nil, false)
_, err = client.Rebuild(ctx, project, "", nil, dirs.OneStep(project), nil, nil, "", nil, false)
if err != nil {
return -1, fmt.Errorf("failed to rebuild continue project %d: %w", project, err)
}
Expand All @@ -430,11 +430,11 @@ func runIteration(ctx context.Context, client *dlc.Client, project int64, operat
}

randomStepVersion := int64(rand.Intn(int(version)))
_, err = client.Rebuild(ctx, project, "", &randomStepVersion, dirs.RandomStep(project), nil, "", nil, false)
_, err = client.Rebuild(ctx, project, "", &randomStepVersion, dirs.RandomStep(project), nil, nil, "", nil, false)
if err != nil {
return -1, fmt.Errorf("failed to rebuild step project %d: %w", project, err)
}
_, err = client.Rebuild(ctx, project, "", &version, dirs.RandomStep(project), nil, "", nil, false)
_, err = client.Rebuild(ctx, project, "", &version, dirs.RandomStep(project), nil, nil, "", nil, false)
if err != nil {
return -1, fmt.Errorf("failed to rebuild step project %d: %w", project, err)
}
Expand Down
25 changes: 23 additions & 2 deletions internal/db/queryBuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func (qb *queryBuilder) updatedObjectsCTE() string {
AND (o.stop_version IS NULL OR o.stop_version > __stop_version__)
%s
%s
%s
ORDER BY o.path
`

Expand All @@ -105,7 +106,12 @@ func (qb *queryBuilder) updatedObjectsCTE() string {
ignoresPredicate = "AND o.path NOT LIKE ALL(__ignores__::text[])"
}

return fmt.Sprintf(template, isCachedSelector, cacheJoin, pathPredicate, ignoresPredicate)
subpathsPredicate := ""
if len(qb.objectQuery.Subpaths) > 0 {
subpathsPredicate = "AND o.path LIKE ALL(__subpaths__::text[])"
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this option isn't mutually exclusive with ignores right -- you could use both at once? And we'll just let PG decide which conditions to use the index for?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use both at once, ignoring a subset of the files under the subpaths. I initially wanted to use it like (haskell's lens)[https://github.com/ekmett/lens/wiki/Overview] functionality where the subpaths focus you on a specific set of the project and let you manipulate just that. With that in mind I thought keeping the ignores made sense.


return fmt.Sprintf(template, isCachedSelector, cacheJoin, pathPredicate, ignoresPredicate, subpathsPredicate)
}

func (qb *queryBuilder) removedObjectsCTE() string {
Expand All @@ -117,6 +123,7 @@ func (qb *queryBuilder) removedObjectsCTE() string {
AND o.stop_version > __start_version__
AND o.stop_version <= __stop_version__
%s
%s
AND NOT (
-- Skip removing files if they are in the updated_objects list
(RIGHT(o.path, 1) != '/' AND o.path IN (SELECT path FROM updated_objects))
Expand All @@ -135,7 +142,12 @@ func (qb *queryBuilder) removedObjectsCTE() string {
ignoresPredicate = "AND o.path NOT LIKE ALL(__ignores__::text[])"
}

return fmt.Sprintf(template, ignoresPredicate)
subpathsPredicate := ""
if len(qb.objectQuery.Subpaths) > 0 {
subpathsPredicate = "AND o.path LIKE ALL(__subpaths__::text[])"
}

return fmt.Sprintf(template, ignoresPredicate, subpathsPredicate)
}

func (qb *queryBuilder) cachedObjectHashesCTE() string {
Expand Down Expand Up @@ -225,6 +237,15 @@ func (qb *queryBuilder) replaceQueryArgs(query string) (string, []any) {
args = append(args, ignorePatterns)
}

if len(qb.objectQuery.Subpaths) > 0 {
argNames = append(argNames, "__subpaths__")
subpathPatterns := []string{}
for _, subPath := range qb.objectQuery.Subpaths {
subpathPatterns = append(subpathPatterns, fmt.Sprintf("%s%%", subPath))
}
args = append(args, subpathPatterns)
}

if len(qb.cacheVersions) > 0 {
argNames = append(argNames, "__cache_versions__")
args = append(args, qb.cacheVersions)
Expand Down
450 changes: 230 additions & 220 deletions internal/pb/fs.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions internal/pb/fs.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ message ObjectQuery {
string path = 1;
bool is_prefix = 2;
repeated string ignores = 4;
repeated string subpaths = 5;
}

message GetRequest {
Expand Down
4 changes: 3 additions & 1 deletion js/src/grpc-client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ export class DateiLagerGrpcClient {
path,
ignores,
isPrefix: true,
subpaths: [],
},
],
},
Expand Down Expand Up @@ -248,7 +249,7 @@ export class DateiLagerGrpcClient {
},
async () => {
const call = this._client.getUnary(
{ project, fromVersion: from, toVersion: to, queries: [{ path, ignores, isPrefix: true }] },
{ project, fromVersion: from, toVersion: to, queries: [{ path, ignores, isPrefix: true, subpaths: [] }] },
this._rpcOptions()
);
return await call.response;
Expand Down Expand Up @@ -286,6 +287,7 @@ export class DateiLagerGrpcClient {
path,
isPrefix: false,
ignores: [],
subpaths: [],
},
],
},
Expand Down
9 changes: 8 additions & 1 deletion pkg/cli/rebuild.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ func NewCmdRebuild() *cobra.Command {
prefix string
dir string
ignores string
subpaths string
summarize bool
cacheDir string
fileMatchInclude string
Expand All @@ -40,12 +41,17 @@ func NewCmdRebuild() *cobra.Command {
ignoreList = strings.Split(ignores, ",")
}

var subpathList []string
if len(subpaths) > 0 {
subpathList = strings.Split(subpaths, ",")
}

matcher, err := files.NewFileMatcher(fileMatchInclude, fileMatchExclude)
if err != nil {
return err
}

result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, cacheDir, matcher, summarize)
result, err := client.Rebuild(ctx, project, prefix, to, dir, ignoreList, subpathList, cacheDir, matcher, summarize)
if err != nil {
return fmt.Errorf("could not rebuild project: %w", err)
}
Expand Down Expand Up @@ -73,6 +79,7 @@ func NewCmdRebuild() *cobra.Command {
cmd.Flags().StringVar(&prefix, "prefix", "", "Search prefix")
cmd.Flags().StringVar(&dir, "dir", "", "Output directory")
cmd.Flags().StringVar(&ignores, "ignores", "", "Comma separated list of ignore paths")
cmd.Flags().StringVar(&subpaths, "subpaths", "", "Comma separated list of subpaths to include")
cmd.Flags().BoolVar(&summarize, "summarize", true, "Should include the summary file (required for future updates)")
cmd.Flags().StringVar(&cacheDir, "cachedir", "", "Path where the cache folder is mounted")
cmd.Flags().StringVar(&fileMatchInclude, "matchinclude", "", "Set fileMatch to true if the written files are matched by this glob pattern")
Expand Down
9 changes: 5 additions & 4 deletions pkg/cli/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ import (

func NewCmdUpdate() *cobra.Command {
var (
project int64
dir string
project int64
dir string
subpaths []string
)

cmd := &cobra.Command{
Expand All @@ -22,7 +23,7 @@ func NewCmdUpdate() *cobra.Command {

client := client.FromContext(ctx)

version, count, err := client.Update(ctx, project, dir)
version, count, err := client.Update(ctx, project, dir, subpaths)
if err != nil {
return fmt.Errorf("update objects: %w", err)
}
Expand All @@ -40,7 +41,7 @@ func NewCmdUpdate() *cobra.Command {

cmd.Flags().Int64Var(&project, "project", -1, "Project ID (required)")
cmd.Flags().StringVar(&dir, "dir", "", "Directory containing updated files")

cmd.Flags().StringSliceVar(&subpaths, "subpaths", nil, "Subpaths to update")
_ = cmd.MarkFlagRequired("project")

return cmd
Expand Down
20 changes: 17 additions & 3 deletions pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,7 @@ func (t *rebuildResultTracker) result() RebuildResult {
}
}

func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVersion *int64, dir string, ignores []string, cacheDir string, matcher *files.FileMatcher, summarize bool) (RebuildResult, error) {
func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVersion *int64, dir string, ignores []string, subpaths []string, cacheDir string, matcher *files.FileMatcher, summarize bool) (RebuildResult, error) {
ctx, span := telemetry.Start(ctx, "client.rebuild", trace.WithAttributes(
key.Project.Attribute(project),
key.Prefix.Attribute(prefix),
Expand All @@ -381,6 +381,7 @@ func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVe
Path: prefix,
IsPrefix: true,
Ignores: ignores,
Subpaths: subpaths,
}

availableCacheVersions := ReadCacheVersionFile(cacheDir)
Expand Down Expand Up @@ -504,7 +505,7 @@ func (c *Client) Rebuild(ctx context.Context, project int64, prefix string, toVe
return result, nil
}

func (c *Client) Update(rootCtx context.Context, project int64, dir string) (int64, uint32, error) {
func (c *Client) Update(rootCtx context.Context, project int64, dir string, subpaths []string) (int64, uint32, error) {
rootCtx, span := telemetry.Start(rootCtx, "client.update", trace.WithAttributes(
key.Project.Attribute(project),
key.Directory.Attribute(dir),
Expand All @@ -525,6 +526,19 @@ func (c *Client) Update(rootCtx context.Context, project int64, dir string) (int
return fromVersion, 0, nil
}

if len(subpaths) > 0 {
filteredUpdates := make([]*fsdiff_pb.Update, 0, len(diff.Updates))
for _, update := range diff.Updates {
for _, subpath := range subpaths {
if strings.HasPrefix(update.Path, subpath) {
filteredUpdates = append(filteredUpdates, update)
break
}
}
}
diff.Updates = filteredUpdates
}

toVersion := int64(-1)

updateChan := make(chan *fsdiff_pb.Update, len(diff.Updates))
Expand Down Expand Up @@ -642,7 +656,7 @@ func (c *Client) Update(rootCtx context.Context, project int64, dir string) (int
return -1, updateCount, err
}
} else {
result, err := c.Rebuild(rootCtx, project, "", nil, dir, nil, "", nil, true)
result, err := c.Rebuild(rootCtx, project, "", nil, dir, nil, nil, "", nil, true)
if err != nil {
return -1, updateCount, err
}
Expand Down
Loading