forked from ultravioletrs/cocos
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
NOISSUE - Enable WASM Support and FileSystem Support (ultravioletrs#189)
* feat(algorithm): Add wasm as an algo type Signed-off-by: Rodney Osodo <[email protected]> * feat(algorithm): Use filesystem to store results Move from unix socket for results storage to filesystem * test: test new filesystem changes Signed-off-by: Rodney Osodo <[email protected]> * refactor(files): rename resultFile to resultsFilePath * feat(wasm-runtime): change from wasmtime to wasmedge Wasmedge enables easier directory mapping to get results Signed-off-by: Rodney Osodo <[email protected]> * feat(algorithm): send results as zipped directory Create a new function to zip the results directory and send it back to the user * fix(wasm): runtime argument Fix the directory mapping for wasm runtime arguments Signed-off-by: Rodney Osodo <[email protected]> * fix(errors): provide useful error message * chore(gitignore): add results zip to gitignore * feat(filesystem): Enable storing results on filesystem for python algos * refactor: revert to upstream cocos repo Signed-off-by: Rodney Osodo <[email protected]> * fix: remove AddDataset from algorithm interface * fix: agent to handle results zipping * test: test zipping directories * refactor(agent): Handle file operations from agent * test: run test inside eos Signed-off-by: Rodney Osodo <[email protected]> * refactor(test): Document and test algos are running Document steps on running the 2 python exampls and ensure they are running on eos Signed-off-by: Rodney Osodo <[email protected]> * fix: remove witheDataset option * test: test without dataset argument Signed-off-by: Rodney Osodo <[email protected]> --------- Signed-off-by: Rodney Osodo <[email protected]>
- Loading branch information
1 parent
3c855e3
commit afc306a
Showing
23 changed files
with
517 additions
and
265 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,5 +10,5 @@ cmd/manager/tmp | |
*.pem | ||
|
||
dist/ | ||
result.bin | ||
result.zip | ||
*.spec |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
// Copyright (c) Ultraviolet | ||
// SPDX-License-Identifier: Apache-2.0 | ||
package algorithm | ||
|
||
import ( | ||
"archive/zip" | ||
"bytes" | ||
"fmt" | ||
"io" | ||
"os" | ||
"path/filepath" | ||
) | ||
|
||
// ZipDirectory zips a directory and returns the zipped bytes. | ||
func ZipDirectory() ([]byte, error) { | ||
buf := new(bytes.Buffer) | ||
zipWriter := zip.NewWriter(buf) | ||
|
||
err := filepath.Walk(ResultsDir, func(path string, info os.FileInfo, err error) error { | ||
if err != nil { | ||
return fmt.Errorf("error walking the path %q: %v", path, err) | ||
} | ||
|
||
if info.IsDir() { | ||
return nil | ||
} | ||
|
||
relPath, err := filepath.Rel(ResultsDir, path) | ||
if err != nil { | ||
return fmt.Errorf("error getting relative path for %q: %v", path, err) | ||
} | ||
|
||
file, err := os.Open(path) | ||
if err != nil { | ||
return fmt.Errorf("error opening file %q: %v", path, err) | ||
} | ||
defer file.Close() | ||
|
||
zipFile, err := zipWriter.Create(relPath) | ||
if err != nil { | ||
return fmt.Errorf("error creating zip file for %q: %v", path, err) | ||
} | ||
|
||
if _, err = io.Copy(zipFile, file); err != nil { | ||
return fmt.Errorf("error copying file %q to zip: %v", path, err) | ||
} | ||
|
||
return err | ||
}) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
if err = zipWriter.Close(); err != nil { | ||
return nil, err | ||
} | ||
|
||
return buf.Bytes(), nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
// Copyright (c) Ultraviolet | ||
// SPDX-License-Identifier: Apache-2.0 | ||
package algorithm_test | ||
|
||
import ( | ||
"os" | ||
"testing" | ||
|
||
"github.com/ultravioletrs/cocos/agent/algorithm" | ||
) | ||
|
||
func TestZipDirectory(t *testing.T) { | ||
cases := []struct { | ||
name string | ||
directories []string | ||
files []string | ||
expected []string | ||
}{ | ||
{ | ||
name: "empty directory", | ||
directories: []string{"testdata"}, | ||
}, | ||
{ | ||
name: "single file", | ||
files: []string{"file1.txt"}, | ||
}, | ||
{ | ||
name: "directory with single file", | ||
directories: []string{"testdata"}, | ||
expected: []string{"testdata/file1.txt"}, | ||
}, | ||
{ | ||
name: "directory with multiple files", | ||
directories: []string{"testdata"}, | ||
expected: []string{ | ||
"testdata/file1.txt", | ||
"testdata/file2.txt", | ||
"testdata/file3.txt", | ||
}, | ||
}, | ||
{ | ||
name: "nested directories", | ||
directories: []string{"testdata", "testdata/nested"}, | ||
expected: []string{ | ||
"testdata/nested/file1.txt", | ||
"testdata/nested/file2.txt", | ||
"testdata/nested/file3.txt", | ||
}, | ||
}, | ||
} | ||
|
||
for _, tc := range cases { | ||
t.Run(tc.name, func(t *testing.T) { | ||
if err := os.Mkdir(algorithm.ResultsDir, 0o755); err != nil { | ||
t.Fatalf("error creating results directory: %s", err.Error()) | ||
} | ||
defer func() { | ||
if err := os.RemoveAll(algorithm.ResultsDir); err != nil { | ||
t.Fatalf("error removing results directory and its contents: %s", err.Error()) | ||
} | ||
}() | ||
|
||
for _, dir := range tc.directories { | ||
if dir != "" { | ||
if err := os.Mkdir(algorithm.ResultsDir+"/"+dir, 0o755); err != nil { | ||
t.Fatalf("error creating test directory: %s", err.Error()) | ||
} | ||
} | ||
} | ||
for _, file := range tc.files { | ||
if _, err := os.Create(algorithm.ResultsDir + "/" + file); err != nil { | ||
t.Fatalf("error creating test file: %s", err.Error()) | ||
} | ||
} | ||
|
||
if _, err := algorithm.ZipDirectory(); err != nil { | ||
t.Errorf("ZipDirectory() error = %v", err) | ||
} | ||
}) | ||
} | ||
} |
Oops, something went wrong.