Merge branch 'bittorrent-v2'

NOTES.md

@@ -30,3 +30,9 @@ The DHT is a bit different: you can't be an active node if you are a badnat, but
 - https://www.bittorrent.org/beps/bep_0055.html
 - https://github.com/anacrolix/torrent/issues/685
 - https://stackoverflow.com/questions/38786438/libutp-%C2%B5tp-and-nat-traversal-udp-hole-punching
+
+### BitTorrent v2
+
+- https://www.bittorrent.org/beps/bep_0052.html
+
+The canonical infohash to use for a torrent will be the v1 infohash, or the short form of the v2 infohash if v1 is not supported. This will apply everywhere that both infohashes are present. If only one 20 byte hash is present, it is always the v1 hash (except in code that interfaces with things that only work with 20 byte hashes, like the DHT).

client.go

@@ -47,6 +47,7 @@ import (
 	"github.com/anacrolix/torrent/storage"
 	"github.com/anacrolix/torrent/tracker"
 	"github.com/anacrolix/torrent/types/infohash"
+	infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2"
 	"github.com/anacrolix/torrent/webtorrent"
 )
 
@@ -155,7 +156,7 @@ func (cl *Client) WriteStatus(_w io.Writer) {
 	fmt.Fprintf(w, "# Torrents: %d\n", len(torrentsSlice))
 	fmt.Fprintln(w)
 	sort.Slice(torrentsSlice, func(l, r int) bool {
-		return torrentsSlice[l].infoHash.AsString() < torrentsSlice[r].infoHash.AsString()
+		return torrentsSlice[l].canonicalShortInfohash().AsString() < torrentsSlice[r].canonicalShortInfohash().AsString()
 	})
 	for _, t := range torrentsSlice {
 		if t.name() == "" {
@@ -305,14 +306,18 @@ func NewClient(cfg *ClientConfig) (cl *Client, err error) {
 	cl.websocketTrackers = websocketTrackers{
 		PeerId: cl.peerID,
 		Logger: cl.logger,
-		GetAnnounceRequest: func(event tracker.AnnounceEvent, infoHash [20]byte) (tracker.AnnounceRequest, error) {
+		GetAnnounceRequest: func(
+			event tracker.AnnounceEvent, infoHash [20]byte,
+		) (
+			tracker.AnnounceRequest, error,
+		) {
 			cl.lock()
 			defer cl.unlock()
 			t, ok := cl.torrents[infoHash]
 			if !ok {
 				return tracker.AnnounceRequest{}, errors.New("torrent not tracked by client")
 			}
-			return t.announceRequest(event), nil
+			return t.announceRequest(event, infoHash), nil
 		},
 		Proxy:                      cl.config.HTTPProxy,
 		WebsocketTrackerHttpHeader: cl.config.WebsocketTrackerHttpHeader,
@@ -902,16 +907,15 @@ func (cl *Client) incomingPeerPort() int {
 	return cl.LocalPort()
 }
 
-func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) error {
+func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) (err error) {
 	if c.headerEncrypted {
 		var rw io.ReadWriter
-		var err error
 		rw, c.cryptoMethod, err = mse.InitiateHandshake(
 			struct {
 				io.Reader
 				io.Writer
 			}{c.r, c.w},
-			t.infoHash[:],
+			t.canonicalShortInfohash().Bytes(),
 			nil,
 			cl.config.CryptoProvides,
 		)
@@ -920,14 +924,19 @@ func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) error {
 			return fmt.Errorf("header obfuscation handshake: %w", err)
 		}
 	}
-	ih, err := cl.connBtHandshake(c, &t.infoHash)
+	ih, err := cl.connBtHandshake(c, t.canonicalShortInfohash())
 	if err != nil {
 		return fmt.Errorf("bittorrent protocol handshake: %w", err)
 	}
-	if ih != t.infoHash {
-		return errors.New("bittorrent protocol handshake: peer infohash didn't match")
+	if g.Some(ih) == t.infoHash {
+		return nil
 	}
-	return nil
+	if t.infoHashV2.Ok && *t.infoHashV2.Value.ToShort() == ih {
+		c.v2 = true
+		return nil
+	}
+	err = errors.New("bittorrent protocol handshake: peer infohash didn't match")
+	return
 }
 
 // Calls f with any secret keys. Note that it takes the Client lock, and so must be used from code
@@ -1284,15 +1293,23 @@ func (cl *Client) newTorrent(ih metainfo.Hash, specStorage storage.ClientImpl) (
 
 // Return a Torrent ready for insertion into a Client.
 func (cl *Client) newTorrentOpt(opts AddTorrentOpts) (t *Torrent) {
+	var v1InfoHash g.Option[infohash.T]
+	if !opts.InfoHash.IsZero() {
+		v1InfoHash.Set(opts.InfoHash)
+	}
+	if !v1InfoHash.Ok && !opts.InfoHashV2.Ok {
+		panic("v1 infohash must be nonzero or v2 infohash must be set")
+	}
 	// use provided storage, if provided
 	storageClient := cl.defaultStorage
 	if opts.Storage != nil {
 		storageClient = storage.NewClient(opts.Storage)
 	}
 
 	t = &Torrent{
-		cl:       cl,
-		infoHash: opts.InfoHash,
+		cl:         cl,
+		infoHash:   v1InfoHash,
+		infoHashV2: opts.InfoHashV2,
 		peers: prioritizedPeers{
 			om: gbtree.New(32),
 			getPrio: func(p PeerInfo) peerPriority {
@@ -1342,10 +1359,13 @@ func (cl *Client) AddTorrentInfoHash(infoHash metainfo.Hash) (t *Torrent, new bo
 	return cl.AddTorrentInfoHashWithStorage(infoHash, nil)
 }
 
-// Adds a torrent by InfoHash with a custom Storage implementation.
+// Deprecated. Adds a torrent by InfoHash with a custom Storage implementation.
 // If the torrent already exists then this Storage is ignored and the
 // existing torrent returned with `new` set to `false`
-func (cl *Client) AddTorrentInfoHashWithStorage(infoHash metainfo.Hash, specStorage storage.ClientImpl) (t *Torrent, new bool) {
+func (cl *Client) AddTorrentInfoHashWithStorage(
+	infoHash metainfo.Hash,
+	specStorage storage.ClientImpl,
+) (t *Torrent, new bool) {
 	cl.lock()
 	defer cl.unlock()
 	t, ok := cl.torrents[infoHash]
@@ -1396,19 +1416,21 @@ func (cl *Client) AddTorrentOpt(opts AddTorrentOpts) (t *Torrent, new bool) {
 }
 
 type AddTorrentOpts struct {
-	InfoHash  infohash.T
-	Storage   storage.ClientImpl
-	ChunkSize pp.Integer
-	InfoBytes []byte
+	InfoHash   infohash.T
+	InfoHashV2 g.Option[infohash_v2.T]
+	Storage    storage.ClientImpl
+	ChunkSize  pp.Integer
+	InfoBytes  []byte
 }
 
 // Add or merge a torrent spec. Returns new if the torrent wasn't already in the client. See also
 // Torrent.MergeSpec.
 func (cl *Client) AddTorrentSpec(spec *TorrentSpec) (t *Torrent, new bool, err error) {
 	t, new = cl.AddTorrentOpt(AddTorrentOpts{
-		InfoHash:  spec.InfoHash,
-		Storage:   spec.Storage,
-		ChunkSize: spec.ChunkSize,
+		InfoHash:   spec.InfoHash,
+		InfoHashV2: spec.InfoHashV2,
+		Storage:    spec.Storage,
+		ChunkSize:  spec.ChunkSize,
 	})
 	modSpec := *spec
 	if new {
@@ -1459,7 +1481,7 @@ func (t *Torrent) MergeSpec(spec *TorrentSpec) error {
 	t.maybeNewConns()
 	t.dataDownloadDisallowed.SetBool(spec.DisallowDataDownload)
 	t.dataUploadDisallowed = spec.DisallowDataUpload
-	return nil
+	return t.AddPieceLayers(spec.PieceLayers)
 }
 
 func (cl *Client) dropTorrent(infoHash metainfo.Hash, wg *sync.WaitGroup) (err error) {

client_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"net/netip"
 	"os"
@@ -131,9 +132,9 @@ func TestAddDropManyTorrents(t *testing.T) {
 	cl, err := NewClient(TestingConfig(t))
 	require.NoError(t, err)
 	defer cl.Close()
-	for i := 0; i < 1000; i += 1 {
+	for i := range 1000 {
 		var spec TorrentSpec
-		binary.PutVarint(spec.InfoHash[:], int64(i))
+		binary.PutVarint(spec.InfoHash[:], int64(i+1))
 		tt, new, err := cl.AddTorrentSpec(&spec)
 		assert.NoError(t, err)
 		assert.True(t, new)
@@ -155,6 +156,7 @@ func TestMergingTrackersByAddingSpecs(t *testing.T) {
 	require.NoError(t, err)
 	defer cl.Close()
 	spec := TorrentSpec{}
+	rand.Read(spec.InfoHash[:])
 	T, new, _ := cl.AddTorrentSpec(&spec)
 	if !new {
 		t.FailNow()
@@ -587,16 +589,20 @@ func TestPeerInvalidHave(t *testing.T) {
 }
 
 func TestPieceCompletedInStorageButNotClient(t *testing.T) {
+	c := qt.New(t)
 	greetingTempDir, greetingMetainfo := testutil.GreetingTestTorrent()
 	defer os.RemoveAll(greetingTempDir)
 	cfg := TestingConfig(t)
 	cfg.DataDir = greetingTempDir
 	seeder, err := NewClient(TestingConfig(t))
 	require.NoError(t, err)
 	defer seeder.Close()
-	seeder.AddTorrentSpec(&TorrentSpec{
+	_, new, err := seeder.AddTorrentSpec(&TorrentSpec{
 		InfoBytes: greetingMetainfo.InfoBytes,
+		InfoHash:  greetingMetainfo.HashInfoBytes(),
 	})
+	c.Check(err, qt.IsNil)
+	c.Check(new, qt.IsTrue)
 }
 
 // Check that when the listen port is 0, all the protocols listened on have

cmd/torrent/serve.go

@@ -44,7 +44,7 @@ func serve() (cmd bargle.Command) {
 			if err != nil {
 				return fmt.Errorf("building info from path %q: %w", filePath, err)
 			}
-			for _, fi := range info.Files {
+			for _, fi := range info.UpvertedFiles() {
 				log.Printf("added %q", fi.Path)
 			}
 			mi := metainfo.MetaInfo{

cmd/torrent2/main.go

@@ -0,0 +1,66 @@
+// This is an alternate to cmd/torrent which has become bloated with awful argument parsing. Since
+// this is my most complicated binary, I will try to build something that satisfies only what I need
+// here.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/anacrolix/torrent/metainfo"
+)
+
+type argError struct {
+	err error
+}
+
+func assertOk(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func bail(str string) {
+	panic(str)
+}
+
+func main() {
+	args := os.Args[1:]
+	map[string]func(){
+		"metainfo": func() {
+			map[string]func(){
+				"validate-v2": func() {
+					mi, err := metainfo.LoadFromFile(args[2])
+					assertOk(err)
+					info, err := mi.UnmarshalInfo()
+					assertOk(err)
+					if !info.HasV2() {
+						bail("not a v2 torrent")
+					}
+					err = metainfo.ValidatePieceLayers(mi.PieceLayers, &info.FileTree, info.PieceLength)
+					assertOk(err)
+				},
+				"pprint": func() {
+					mi, err := metainfo.LoadFromFile(args[2])
+					assertOk(err)
+					info, err := mi.UnmarshalInfo()
+					assertOk(err)
+					files := info.UpvertedFiles()
+					pieceIndex := 0
+					for _, f := range files {
+						numPieces := int((f.Length + info.PieceLength - 1) / info.PieceLength)
+						endIndex := pieceIndex + numPieces
+						fmt.Printf(
+							"%x: %q: pieces (%v-%v)\n",
+							f.PiecesRoot.Unwrap(),
+							f.BestPath(),
+							pieceIndex,
+							endIndex-1,
+						)
+						pieceIndex = endIndex
+					}
+				},
+			}[args[1]]()
+		},
+	}[args[0]]()
+}

common/upverted_files.go

@@ -16,3 +16,12 @@ func LengthIterFromUpvertedFiles(fis []metainfo.FileInfo) segments.LengthIter {
 		return l, true
 	}
 }
+
+// Returns file segments, BitTorrent v2 aware.
+func TorrentOffsetFileSegments(info *metainfo.Info) (ret []segments.Extent) {
+	files := info.UpvertedFiles()
+	for _, fi := range files {
+		ret = append(ret, segments.Extent{fi.TorrentOffset, fi.Length})
+	}
+	return
+}

file.go

@@ -1,7 +1,10 @@
 package torrent
 
 import (
+	"crypto/sha256"
+
 	"github.com/RoaringBitmap/roaring"
+	g "github.com/anacrolix/generics"
 	"github.com/anacrolix/missinggo/v2/bitmap"
 
 	"github.com/anacrolix/torrent/metainfo"
@@ -16,6 +19,11 @@ type File struct {
 	fi          metainfo.FileInfo
 	displayPath string
 	prio        piecePriority
+	piecesRoot  g.Option[[sha256.Size]byte]
+}
+
+func (f *File) String() string {
+	return f.Path()
 }
 
 func (f *File) Torrent() *Torrent {
@@ -28,12 +36,12 @@ func (f *File) Offset() int64 {
 }
 
 // The FileInfo from the metainfo.Info to which this file corresponds.
-func (f File) FileInfo() metainfo.FileInfo {
+func (f *File) FileInfo() metainfo.FileInfo {
 	return f.fi
 }
 
 // The file's path components joined by '/'.
-func (f File) Path() string {
+func (f *File) Path() string {
 	return f.path
 }
 
@@ -204,3 +212,7 @@ func (f *File) EndPieceIndex() int {
 	}
 	return pieceIndex((f.offset + f.length + int64(f.t.usualPieceSize()) - 1) / int64(f.t.usualPieceSize()))
 }
+
+func (f *File) numPieces() int {
+	return f.EndPieceIndex() - f.BeginPieceIndex()
+}

fs/torrentfs.go

@@ -72,7 +72,7 @@ func isSubPath(parent, child string) bool {
 
 func (dn dirNode) ReadDirAll(ctx context.Context) (des []fuse.Dirent, err error) {
 	names := map[string]bool{}
-	for _, fi := range dn.metadata.Files {
+	for _, fi := range dn.metadata.UpvertedFiles() {
 		filePathname := strings.Join(fi.Path, "/")
 		if !isSubPath(dn.path, filePathname) {
 			continue