Merge pull request #73 from SuperclusterLabs/govi218/patch-file-api

Patch file listing API, add route to delete all pins

go.mod

@@ -22,12 +22,14 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-cmds v0.8.1 // indirect
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
+	github.com/rs/cors v1.7.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect
 	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
@@ -91,13 +93,13 @@ require (
 	github.com/ipfs/go-filestore v1.2.0 // indirect
 	github.com/ipfs/go-fs-lock v0.0.7 // indirect
 	github.com/ipfs/go-graphsync v0.13.1 // indirect
-	github.com/ipfs/go-ipfs-api v0.3.0
 	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
 	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
 	github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
 	github.com/ipfs/go-ipfs-files v0.1.1
+	github.com/ipfs/go-ipfs-http-client v0.4.0
 	github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
 	github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
 	github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect

go.sum

@@ -598,6 +598,7 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
+github.com/ipfs/go-ipfs-cmds v0.8.1 h1:El661DBWqdqwgz7B9xwKyUpigwqk6BBBHb5B8DfJP00=
 github.com/ipfs/go-ipfs-cmds v0.8.1/go.mod h1:y0bflH6m4g6ary4HniYt98UqbrVnRxmRarzeMdLIUn0=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
@@ -620,6 +621,8 @@ github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTi
 github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84=
 github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
 github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
+github.com/ipfs/go-ipfs-http-client v0.4.0 h1:LNuVbFoKfCohCmcNImml3byM3PpTxTT7RPrv/UoDFkI=
+github.com/ipfs/go-ipfs-http-client v0.4.0/go.mod h1:NXzPUKt/QVCuR74a8angJCGOSLPImNi5LqaTxIep/70=
 github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
 github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
 github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=
@@ -1515,6 +1518,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM=

main.go

@@ -42,7 +42,7 @@ func main() {
 		IpfsDaemon: ipfs,
 		Engine:     gin.Default(),
 	}
-	s := store.NewIpfsStore()
+	s, err := store.NewIpfsStore()
 	if err != nil {
 		panic("Cannot create store: " + err.Error())
 	}

model/file.go

@@ -4,6 +4,8 @@ type File struct {
 	Name      string `json:"name"`
 	Cid       string `json:"cid"`
 	Creator   string `json:"creator"`
-	CreatedAt int64  `json:"createdAt"`
+	CreatedAt int64  `json:"-"`
+	Size      int64  `json:"size"`
+	PinType   string `json:"pinType"`
 	// TODO: whitelist of users that can access
 }

router/api.go

@@ -96,6 +96,17 @@ func deleteFile(ctx *gin.Context, s store.P2PStore) {
 	ctx.Status(http.StatusOK)
 }
 
+func deleteAll(ctx *gin.Context, s store.P2PStore) {
+	err := s.DeleteAll(ctx)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, ResponseError{
+			Error: err.Error(),
+		})
+		return
+	}
+	ctx.Status(http.StatusOK)
+}
+
 func modifyFile(ctx *gin.Context, s store.P2PStore) {
 	name := ctx.Param("name")
 

router/routes.go

@@ -52,6 +52,7 @@ func AddRoutes(r proc.SuperclusterRuntime, s store.P2PStore) {
 	// cluster API
 	api.POST("/cluster", func(ctx *gin.Context) { createCluster(ctx) })
 	api.GET("/cluster/files", func(ctx *gin.Context) { listPinnedFiles(ctx, s) })
+	api.DELETE("/cluster/files", func(ctx *gin.Context) { deleteAll(ctx, s) })
 
 	api.GET("/cluster/:clusterId", func(ctx *gin.Context) { getCluster(ctx) })
 	api.PUT("/cluster/:clusterId", func(ctx *gin.Context) { modifyCluster(ctx) })

router/user_api.go

@@ -162,7 +162,7 @@ func connectPeer(ctx *gin.Context, s store.P2PStore) {
 		return
 	}
 
-	err := s.ConnectPeer(ctx, a.Addrs...)
+	err := s.ConnectPeer(ctx, a.Addresses...)
 	if err != nil {
 		ctx.JSON(http.StatusInternalServerError, ResponseError{
 			Error: err.Error(),
@@ -178,6 +178,7 @@ func getAddrs(ctx *gin.Context, s store.P2PStore) {
 		ctx.JSON(http.StatusInternalServerError, ResponseError{
 			Error: err.Error(),
 		})
+		return
 	}
 
 	ctx.JSON(http.StatusOK, info)

store/ipfs_store.go

@@ -1,64 +1,78 @@
 package store
 
 import (
-	"bytes"
 	"context"
+	"encoding/json"
 	"log"
-	"sort"
-	"time"
+	"net/http"
 
 	"github.com/SuperclusterLabs/supercluster-client/model"
 
 	"github.com/gin-gonic/gin"
-	shell "github.com/ipfs/go-ipfs-api"
+	ipfsFiles "github.com/ipfs/go-ipfs-files"
+	shell "github.com/ipfs/go-ipfs-http-client"
+	path "github.com/ipfs/interface-go-ipfs-core/path"
+	"github.com/libp2p/go-libp2p-core/peer"
+	ma "github.com/multiformats/go-multiaddr"
 )
 
 // TODO: we should transition this to (what?) metadata
 type IpfsStore struct {
 	// abstraction that maps file names to file structs
 	// reqd for file modifications, etc
 	files   map[string]*model.File
-	ipfsApi *shell.Shell
+	ipfsApi *shell.HttpApi
 }
 
 var _ P2PStore = (*IpfsStore)(nil)
 
-func NewIpfsStore() P2PStore {
+func NewIpfsStore() (P2PStore, error) {
+	api, err := shell.NewURLApiWithClient("localhost:5001", &http.Client{
+		Transport: &http.Transport{
+			Proxy:             http.ProxyFromEnvironment,
+			DisableKeepAlives: true,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
 	s := &IpfsStore{
 		files:   make(map[string]*model.File),
-		ipfsApi: shell.NewShell("localhost:5001"),
+		ipfsApi: api,
 	}
 
-	return s
+	return s, nil
 }
 
 func (s *IpfsStore) Create(ctx *gin.Context, name string, contents []byte) (*model.File, error) {
-	// Allow the same file to be pinned again for now, no harm
-	// if _, ok := s.files[name]; ok {
-	// log.Println("Could not create file: ", ErrFileExists.Error())
-	// return nil, ErrFileExists
-	// }
-
-	cid, err := s.ipfsApi.Add(bytes.NewReader(contents))
+	// This is a hack to track metadata for a file. Since a dir is a file
+	// containing file info, we can use it to track file metadata.
+	// N.B: IPFS only stores name, size (bytes), and cid
+	f := ipfsFiles.NewBytesFile(contents)
+	cid, err := s.ipfsApi.Unixfs().
+		Add(ctx, ipfsFiles.NewMapDirectory(map[string]ipfsFiles.Node{
+			name: f,
+		}))
 	if err != nil {
 		log.Println("Could not create file: ", err.Error())
 		return nil, err
 	}
 
-	// TODO: does adding above automatically pin? Do we only need
-	// one of these 2 calls?
-	err = s.ipfsApi.Pin(cid)
+	err = s.ipfsApi.Pin().Add(ctx, cid)
 	if err != nil {
-		log.Println("Could not create file: ", err.Error())
+		log.Println("Could not pin file: ", err.Error())
 		return nil, err
 	}
 
+	// TODO: figure out a way to embed created time/creator info
+	// into ipfs file description
 	new := &model.File{
-		Cid:       cid,
-		Name:      name,
-		CreatedAt: time.Now().Unix(),
+		Cid:  cid.Cid().String(),
+		Name: name,
+		Size: int64(len(contents)),
+		// TODO: is pin type only one of 2 options?
+		PinType: "recursive",
 	}
-	s.files[name] = new
 
 	return new, nil
 }
@@ -67,50 +81,107 @@ func (s *IpfsStore) Modify(ctx context.Context, name, contents string) (*model.F
 	return nil, nil
 }
 
-func (s *IpfsStore) Delete(ctx context.Context, cId string) error {
-	// f := s.files[name]
-	err := s.ipfsApi.Unpin(cId)
+func (s *IpfsStore) Delete(ctx context.Context, cid string) error {
+	p := path.New(cid)
+	err := s.ipfsApi.Pin().Rm(ctx, p)
 	if err != nil {
 		log.Println("Could not remove file ", err.Error())
 		return err
 	}
 
-	// delete(s.files, name)
+	return nil
+}
+
+func (s *IpfsStore) DeleteAll(ctx context.Context) error {
+	fs, err := s.List(ctx)
+	if err != nil {
+		log.Println("Could not fetch pinned files ", err.Error())
+		return err
+	}
+
+	for _, f := range fs {
+		if f.PinType == "recursive" {
+			err := s.ipfsApi.Pin().Rm(ctx, path.New(f.Cid))
+			if err != nil {
+				return err
+			}
+		}
+	}
 
 	return nil
 }
 
 func (s *IpfsStore) List(ctx context.Context) ([]model.File, error) {
 	files := make([]model.File, 0)
 
-	for _, f := range s.files {
-		files = append(files, *f)
+	pins, err := s.ipfsApi.Pin().Ls(ctx)
+	if err != nil {
+		return nil, err
 	}
 
-	sort.Slice(files, func(i, j int) bool {
-		return files[i].CreatedAt < files[j].CreatedAt
-	})
+	// since all files are directories, grab name from them
+	for p := range pins {
+		dir := false
+		es, err := s.ipfsApi.Unixfs().Ls(ctx, p.Path())
+		if err != nil {
+			return nil, err
+		}
+
+		// ignore indirect pins as they are necessarily files for now
+
+		for e := range es {
+			dir = true
+			files = append(files, model.File{
+				Cid:     e.Cid.String(),
+				Name:    e.Name,
+				Size:    int64(e.Size),
+				PinType: "indirect",
+			})
+		}
+
+		if dir {
+			files = append(files, model.File{
+				Cid:     p.Path().Cid().String(),
+				PinType: p.Type(),
+			})
+		}
+	}
 
 	return files, nil
 }
 
 func (s *IpfsStore) GetInfo(ctx context.Context) (*P2PNodeInfo, error) {
-	n, err := s.ipfsApi.ID()
+	resp, err := http.Post("http://localhost:5001/api/v0/id", "application/json", nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	var ar P2PNodeInfo
+	err = json.NewDecoder(resp.Body).Decode(&ar)
 	if err != nil {
 		return nil, err
 	}
-	return &P2PNodeInfo{
-		ID:    n.ID,
-		Addrs: n.Addresses,
-	}, nil
+
+	return &ar, nil
 }
 
 func (s *IpfsStore) PinFile(ctx *gin.Context, c string) error {
-	err := s.ipfsApi.Pin(c)
+	err := s.ipfsApi.Pin().Add(ctx, path.New(c))
 	return err
 }
 
-func (s *IpfsStore) ConnectPeer(ctx *gin.Context, addr ...string) error {
-	err := s.ipfsApi.SwarmConnect(ctx, addr...)
+func (s *IpfsStore) ConnectPeer(ctx *gin.Context, addrs ...string) error {
+	var ms []ma.Multiaddr
+	for _, a := range addrs {
+		m, err := ma.NewMultiaddr(a)
+		if err != nil {
+			return err
+		}
+
+		ms = append(ms, m)
+	}
+	err := s.ipfsApi.Swarm().Connect(ctx, peer.AddrInfo{
+		Addrs: ms,
+	})
 	return err
 }

store/os_store.go

@@ -93,7 +93,24 @@ func (s *osStore) Delete(ctx context.Context, name string) error {
 		log.Println("Could not delete file: ", err.Error())
 		return err
 	}
-	delete(s.files, name)
+	return nil
+}
+
+func (s *osStore) DeleteAll(ctx context.Context) error {
+	fs, err := s.List(ctx)
+	if err != nil {
+		log.Println("Could not fetch pinned files ", err.Error())
+		return err
+	}
+
+	for _, f := range fs {
+		err := os.Remove(StoreName + "/" + f.Name)
+		if err != nil {
+			log.Println("Could not delete file: ", err.Error())
+			return err
+		}
+	}
+
 	return nil
 }
 

store/store.go

@@ -12,13 +12,17 @@ type P2PStore interface {
 	Create(ctx *gin.Context, name string, contents []byte) (*model.File, error)
 	Modify(ctx context.Context, name, contents string) (*model.File, error)
 	Delete(ctx context.Context, name string) error
+	DeleteAll(ctx context.Context) error
 	List(ctx context.Context) ([]model.File, error)
 	GetInfo(ctx context.Context) (*P2PNodeInfo, error)
 	PinFile(ctx *gin.Context, c string) error
 	ConnectPeer(ctx *gin.Context, addr ...string) error
 }
 
 type P2PNodeInfo struct {
-	ID    string   `json:"id"`
-	Addrs []string `json:"addrs"`
+	ID              string
+	PublicKey       string
+	Addresses       []string
+	AgentVersion    string
+	ProtocolVersion string
 }