From ac62a81efc5fdc0de7a99e66bc9a88eb73eadb50 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 27 Feb 2024 11:06:17 +1100 Subject: [PATCH 01/15] Add MagnetV2 and infohash_v2 --- go.mod | 7 ++ go.sum | 18 +++- metainfo/magnet-v2.go | 147 +++++++++++++++++++++++++++++++ metainfo/magnet-v2_test.go | 38 ++++++++ metainfo/magnet.go | 59 ++++++++----- metainfo/magnet_test.go | 21 +++++ types/infohash-v2/infohash-v2.go | 95 ++++++++++++++++++++ types/types.go | 4 +- 8 files changed, 364 insertions(+), 25 deletions(-) create mode 100644 metainfo/magnet-v2.go create mode 100644 metainfo/magnet-v2_test.go create mode 100644 types/infohash-v2/infohash-v2.go diff --git a/go.mod b/go.mod index ce02313504..075c2bddc5 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/gorilla/websocket v1.5.0 github.com/jessevdk/go-flags v1.5.0 + github.com/multiformats/go-multihash v0.2.3 github.com/pion/datachannel v1.5.2 github.com/pion/logging v0.2.2 github.com/pion/webrtc/v3 v3.1.42 @@ -74,11 +75,15 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect + github.com/multiformats/go-varint v0.0.6 // indirect github.com/pion/dtls/v2 v2.2.4 // indirect github.com/pion/ice/v2 v2.2.6 // indirect github.com/pion/interceptor v0.1.11 // indirect @@ -102,6 +107,7 @@ require ( github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 // indirect go.opentelemetry.io/proto/otlp v0.18.0 // indirect @@ -114,6 +120,7 @@ require ( google.golang.org/grpc v1.56.3 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.1.6 // indirect modernc.org/libc v1.22.3 // indirect modernc.org/mathutil v1.5.0 // indirect modernc.org/memory v1.5.0 // indirect diff --git a/go.sum b/go.sum index 75919e87ad..d371109d1d 100644 --- a/go.sum +++ b/go.sum @@ -332,6 +332,10 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -348,14 +352,22 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -477,8 +489,9 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -707,6 +720,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -904,6 +918,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= +lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY= modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= diff --git a/metainfo/magnet-v2.go b/metainfo/magnet-v2.go new file mode 100644 index 0000000000..a6c2c8b4d4 --- /dev/null +++ b/metainfo/magnet-v2.go @@ -0,0 +1,147 @@ +package metainfo + +import ( + "encoding/hex" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/multiformats/go-multihash" + + g "github.com/anacrolix/generics" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" +) + +// Magnet link components. +type MagnetV2 struct { + InfoHash g.Option[Hash] // Expected in this implementation + V2InfoHash g.Option[infohash_v2.T] + Trackers []string // "tr" values + DisplayName string // "dn" value, if not empty + Params url.Values // All other values, such as "x.pe", "as", "xs" etc. +} + +const ( + btmhPrefix = "urn:btmh:" +) + +func (m MagnetV2) String() string { + // Deep-copy m.Params + vs := make(url.Values, len(m.Params)+len(m.Trackers)+2) + for k, v := range m.Params { + vs[k] = append([]string(nil), v...) + } + + for _, tr := range m.Trackers { + vs.Add("tr", tr) + } + if m.DisplayName != "" { + vs.Add("dn", m.DisplayName) + } + + // Transmission and Deluge both expect "urn:btih:" to be unescaped. Deluge wants it to be at the + // start of the magnet link. The InfoHash field is expected to be BitTorrent in this + // implementation. + u := url.URL{ + Scheme: "magnet", + } + var queryParts []string + if m.InfoHash.Ok { + queryParts = append(queryParts, "xt="+btihPrefix+m.InfoHash.Value.HexString()) + } + if m.V2InfoHash.Ok { + queryParts = append( + queryParts, + "xt="+btmhPrefix+infohash_v2.ToMultihash(m.V2InfoHash.Value).HexString(), + ) + } + if rem := vs.Encode(); rem != "" { + queryParts = append(queryParts, rem) + } + u.RawQuery = strings.Join(queryParts, "&") + return u.String() +} + +// ParseMagnetUri parses Magnet-formatted URIs into a Magnet instance +func ParseMagnetV2Uri(uri string) (m MagnetV2, err error) { + u, err := url.Parse(uri) + if err != nil { + err = fmt.Errorf("error parsing uri: %w", err) + return + } + if u.Scheme != "magnet" { + err = fmt.Errorf("unexpected scheme %q", u.Scheme) + return + } + q := u.Query() + for _, xt := range q["xt"] { + if hashStr, found := strings.CutPrefix(xt, btihPrefix); found { + if m.InfoHash.Ok { + err = errors.New("more than one infohash found in magnet link") + return + } + m.InfoHash.Value, err = parseEncodedV1Infohash(hashStr) + if err != nil { + err = fmt.Errorf("error parsing infohash %q: %w", hashStr, err) + return + } + m.InfoHash.Ok = true + } else if hashStr, found := strings.CutPrefix(xt, btmhPrefix); found { + if m.V2InfoHash.Ok { + err = errors.New("more than one infohash found in magnet link") + return + } + m.V2InfoHash.Value, err = parseV2Infohash(hashStr) + if err != nil { + err = fmt.Errorf("error parsing infohash %q: %w", hashStr, err) + return + } + m.V2InfoHash.Ok = true + } else { + lazyAddParam(&m.Params, "xt", xt) + } + } + q.Del("xt") + m.DisplayName = popFirstValue(q, "dn").UnwrapOrZeroValue() + m.Trackers = q["tr"] + q.Del("tr") + // Add everything we haven't consumed. + copyParams(&m.Params, q) + return +} + +func lazyAddParam(vs *url.Values, k, v string) { + if *vs == nil { + g.MakeMap(vs) + } + vs.Add(k, v) +} + +func copyParams(dest *url.Values, src url.Values) { + for k, vs := range src { + for _, v := range vs { + lazyAddParam(dest, k, v) + } + } +} + +func parseV2Infohash(encoded string) (ih infohash_v2.T, err error) { + b, err := hex.DecodeString(encoded) + if err != nil { + return + } + mh, err := multihash.Decode(b) + if err != nil { + return + } + if mh.Code != multihash.SHA2_256 || mh.Length != infohash_v2.Size || len(mh.Digest) != infohash_v2.Size { + err = errors.New("bad multihash") + return + } + n := copy(ih[:], mh.Digest) + if n != infohash_v2.Size { + panic(n) + } + return +} diff --git a/metainfo/magnet-v2_test.go b/metainfo/magnet-v2_test.go new file mode 100644 index 0000000000..620d385c66 --- /dev/null +++ b/metainfo/magnet-v2_test.go @@ -0,0 +1,38 @@ +package metainfo + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestParseMagnetV2(t *testing.T) { + c := qt.New(t) + + const v2Only = "magnet:?xt=urn:btmh:1220caf1e1c30e81cb361b9ee167c4aa64228a7fa4fa9f6105232b28ad099f3a302e&dn=bittorrent-v2-test" + + m2, err := ParseMagnetV2Uri(v2Only) + c.Assert(err, qt.IsNil) + c.Check(m2.InfoHash.Ok, qt.IsFalse) + c.Check(m2.V2InfoHash.Ok, qt.IsTrue) + c.Check(m2.V2InfoHash.Value.HexString(), qt.Equals, "caf1e1c30e81cb361b9ee167c4aa64228a7fa4fa9f6105232b28ad099f3a302e") + c.Check(m2.Params, qt.HasLen, 0) + + _, err = ParseMagnetUri(v2Only) + c.Check(err, qt.IsNotNil) + + const hybrid = "magnet:?xt=urn:btih:631a31dd0a46257d5078c0dee4e66e26f73e42ac&xt=urn:btmh:1220d8dd32ac93357c368556af3ac1d95c9d76bd0dff6fa9833ecdac3d53134efabb&dn=bittorrent-v1-v2-hybrid-test" + + m2, err = ParseMagnetV2Uri(hybrid) + c.Assert(err, qt.IsNil) + c.Check(m2.InfoHash.Ok, qt.IsTrue) + c.Check(m2.InfoHash.Value.HexString(), qt.Equals, "631a31dd0a46257d5078c0dee4e66e26f73e42ac") + c.Check(m2.V2InfoHash.Ok, qt.IsTrue) + c.Check(m2.V2InfoHash.Value.HexString(), qt.Equals, "d8dd32ac93357c368556af3ac1d95c9d76bd0dff6fa9833ecdac3d53134efabb") + c.Check(m2.Params, qt.HasLen, 0) + + m, err := ParseMagnetUri(hybrid) + c.Assert(err, qt.IsNil) + c.Check(m.InfoHash.HexString(), qt.Equals, "631a31dd0a46257d5078c0dee4e66e26f73e42ac") + c.Check(m.Params["xt"], qt.HasLen, 1) +} diff --git a/metainfo/magnet.go b/metainfo/magnet.go index 48dc148e87..7916f9a7fa 100644 --- a/metainfo/magnet.go +++ b/metainfo/magnet.go @@ -7,6 +7,10 @@ import ( "fmt" "net/url" "strings" + + g "github.com/anacrolix/generics" + + "github.com/anacrolix/torrent/types/infohash" ) // Magnet link components. @@ -17,7 +21,7 @@ type Magnet struct { Params url.Values // All other values, such as "x.pe", "as", "xs" etc. } -const xtPrefix = "urn:btih:" +const btihPrefix = "urn:btih:" func (m Magnet) String() string { // Deep-copy m.Params @@ -38,7 +42,7 @@ func (m Magnet) String() string { // implementation. u := url.URL{ Scheme: "magnet", - RawQuery: "xt=" + xtPrefix + m.InfoHash.HexString(), + RawQuery: "xt=" + btihPrefix + m.InfoHash.HexString(), } if len(vs) != 0 { u.RawQuery += "&" + vs.Encode() @@ -61,30 +65,37 @@ func ParseMagnetUri(uri string) (m Magnet, err error) { return } q := u.Query() - xt := q.Get("xt") - m.InfoHash, err = parseInfohash(q.Get("xt")) - if err != nil { - err = fmt.Errorf("error parsing infohash %q: %w", xt, err) + gotInfohash := false + for _, xt := range q["xt"] { + if gotInfohash { + lazyAddParam(&m.Params, "xt", xt) + continue + } + encoded, found := strings.CutPrefix(xt, btihPrefix) + if !found { + lazyAddParam(&m.Params, "xt", xt) + continue + } + m.InfoHash, err = parseEncodedV1Infohash(encoded) + if err != nil { + err = fmt.Errorf("error parsing v1 infohash %q: %w", xt, err) + return + } + gotInfohash = true + } + if !gotInfohash { + err = errors.New("missing v1 infohash") return } - dropFirst(q, "xt") - m.DisplayName = q.Get("dn") - dropFirst(q, "dn") + q.Del("xt") + m.DisplayName = popFirstValue(q, "dn").UnwrapOrZeroValue() m.Trackers = q["tr"] - delete(q, "tr") - if len(q) == 0 { - q = nil - } - m.Params = q + q.Del("tr") + copyParams(&m.Params, q) return } -func parseInfohash(xt string) (ih Hash, err error) { - if !strings.HasPrefix(xt, xtPrefix) { - err = errors.New("bad xt parameter prefix") - return - } - encoded := xt[len(xtPrefix):] +func parseEncodedV1Infohash(encoded string) (ih infohash.T, err error) { decode := func() func(dst, src []byte) (int, error) { switch len(encoded) { case 40: @@ -109,12 +120,16 @@ func parseInfohash(xt string) (ih Hash, err error) { return } -func dropFirst(vs url.Values, key string) { +func popFirstValue(vs url.Values, key string) g.Option[string] { sl := vs[key] switch len(sl) { - case 0, 1: + case 0: + return g.None[string]() + case 1: vs.Del(key) + return g.Some(sl[0]) default: vs[key] = sl[1:] + return g.Some(sl[0]) } } diff --git a/metainfo/magnet_test.go b/metainfo/magnet_test.go index 24ab15b18e..ba13ac8857 100644 --- a/metainfo/magnet_test.go +++ b/metainfo/magnet_test.go @@ -2,6 +2,8 @@ package metainfo import ( "encoding/hex" + "github.com/davecgh/go-spew/spew" + qt "github.com/frankban/quicktest" "testing" "github.com/stretchr/testify/assert" @@ -112,3 +114,22 @@ func contains(haystack []string, needle string) bool { } return false } + +// Check that we can parse the magnet link generated from a real-world torrent. This was added due +// to a regression in copyParams. +func TestParseSintelMagnet(t *testing.T) { + c := qt.New(t) + mi, err := LoadFromFile("../testdata/sintel.torrent") + c.Assert(err, qt.IsNil) + m := mi.Magnet(nil, nil) + ms := m.String() + t.Logf("magnet link: %q", ms) + m, err = ParseMagnetUri(ms) + c.Check(err, qt.IsNil) + spewCfg := spew.NewDefaultConfig() + spewCfg.DisableMethods = true + spewCfg.Dump(m) + m2, err := ParseMagnetV2Uri(ms) + spewCfg.Dump(m2) + c.Check(err, qt.IsNil) +} diff --git a/types/infohash-v2/infohash-v2.go b/types/infohash-v2/infohash-v2.go new file mode 100644 index 0000000000..02ddd1d8f2 --- /dev/null +++ b/types/infohash-v2/infohash-v2.go @@ -0,0 +1,95 @@ +package infohash_v2 + +import ( + "crypto/sha256" + "encoding" + "encoding/hex" + "fmt" + + "github.com/multiformats/go-multihash" + + "github.com/anacrolix/torrent/types/infohash" +) + +const Size = sha256.Size + +// 32-byte SHA2-256 hash. See BEP 52. +type T [Size]byte + +var _ fmt.Formatter = (*T)(nil) + +func (t *T) Format(f fmt.State, c rune) { + // TODO: I can't figure out a nice way to just override the 'x' rune, since it's meaningless + // with the "default" 'v', or .String() already returning the hex. + f.Write([]byte(t.HexString())) +} + +func (t *T) Bytes() []byte { + return t[:] +} + +func (t *T) AsString() string { + return string(t[:]) +} + +func (t *T) String() string { + return t.HexString() +} + +func (t *T) HexString() string { + return fmt.Sprintf("%x", t[:]) +} + +func (t *T) FromHexString(s string) (err error) { + if len(s) != 2*Size { + err = fmt.Errorf("hash hex string has bad length: %d", len(s)) + return + } + n, err := hex.Decode(t[:], []byte(s)) + if err != nil { + return + } + if n != Size { + panic(n) + } + return +} + +// Truncates the hash to 20 bytes for use in auxiliary interfaces, like DHT and trackers. +func (t *T) ToShort() (short infohash.T) { + copy(short[:], t[:]) + return +} + +var ( + _ encoding.TextUnmarshaler = (*T)(nil) + _ encoding.TextMarshaler = T{} +) + +func (t *T) UnmarshalText(b []byte) error { + return t.FromHexString(string(b)) +} + +func (t T) MarshalText() (text []byte, err error) { + return []byte(t.HexString()), nil +} + +func FromHexString(s string) (h T) { + err := h.FromHexString(s) + if err != nil { + panic(err) + } + return +} + +func HashBytes(b []byte) (ret T) { + hasher := sha256.New() + hasher.Write(b) + copy(ret[:], hasher.Sum(nil)) + return +} + +func ToMultihash(t T) multihash.Multihash { + b, _ := multihash.Encode(t[:], multihash.SHA2_256) + return b +} diff --git a/types/types.go b/types/types.go index a06f7e6a8d..8ec7aedfc6 100644 --- a/types/types.go +++ b/types/types.go @@ -45,8 +45,8 @@ const ( PiecePriorityNormal // Wanted. PiecePriorityHigh // Wanted a lot. PiecePriorityReadahead // May be required soon. - // Succeeds a piece where a read occurred. Currently the same as Now, - // apparently due to issues with caching. + // Succeeds a piece where a read occurred. Currently, the same as Now, apparently due to issues + // with caching. PiecePriorityNext PiecePriorityNow // A Reader is reading in this piece. Highest urgency. ) From 7f18ac8f1663c6101c57c9316acaca49c5363261 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 26 Feb 2024 09:28:58 +1100 Subject: [PATCH 02/15] Add BEP 47 extended file attributes fields --- metainfo/bep47.go | 8 ++++++++ metainfo/fileinfo.go | 8 ++++++-- metainfo/info.go | 5 +++-- 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 metainfo/bep47.go diff --git a/metainfo/bep47.go b/metainfo/bep47.go new file mode 100644 index 0000000000..884d88d65a --- /dev/null +++ b/metainfo/bep47.go @@ -0,0 +1,8 @@ +package metainfo + +// See BEP 47. This is common to both Info and FileInfo. +type ExtendedFileAttrs struct { + Attr string `bencode:"attr,omitempty"` + SymlinkPath []string `bencode:"symlink path,omitempty"` + Sha1 string `bencode:"sha1,omitempty"` +} diff --git a/metainfo/fileinfo.go b/metainfo/fileinfo.go index 2a5ea01d1f..894018c108 100644 --- a/metainfo/fileinfo.go +++ b/metainfo/fileinfo.go @@ -4,9 +4,13 @@ import "strings" // Information specific to a single file inside the MetaInfo structure. type FileInfo struct { - Length int64 `bencode:"length"` // BEP3 - Path []string `bencode:"path"` // BEP3 + // BEP3. With BEP 47 this can be optional, but we have no way to describe that without breaking + // the API. + Length int64 `bencode:"length"` + Path []string `bencode:"path"` // BEP3 PathUtf8 []string `bencode:"path.utf-8,omitempty"` + + ExtendedFileAttrs } func (fi *FileInfo) DisplayPath(info *Info) string { diff --git a/metainfo/info.go b/metainfo/info.go index 5eedfa3b4c..b5e4d6e179 100644 --- a/metainfo/info.go +++ b/metainfo/info.go @@ -17,8 +17,9 @@ type Info struct { Pieces []byte `bencode:"pieces"` // BEP3 Name string `bencode:"name"` // BEP3 NameUtf8 string `bencode:"name.utf-8,omitempty"` - Length int64 `bencode:"length,omitempty"` // BEP3, mutually exclusive with Files - Private *bool `bencode:"private,omitempty"` // BEP27 + Length int64 `bencode:"length,omitempty"` // BEP3, mutually exclusive with Files + ExtendedFileAttrs + Private *bool `bencode:"private,omitempty"` // BEP27 // TODO: Document this field. Source string `bencode:"source,omitempty"` Files []FileInfo `bencode:"files,omitempty"` // BEP3, mutually exclusive with Length From eaaa9c0b82ca42a14951532ed4ad2799894fcf4f Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 26 Feb 2024 17:35:14 +1100 Subject: [PATCH 03/15] Merkle hashing and v2 file handling --- NOTES.md | 6 +++ client.go | 24 ++++++---- cmd/torrent2/main.go | 44 ++++++++++++++++++ file.go | 15 +++++- go.mod | 4 +- merkle/merkle.go | 42 +++++++++++++++++ metainfo/bep52.go | 58 +++++++++++++++++++++++ metainfo/file-tree.go | 97 +++++++++++++++++++++++++++++++++++++++ metainfo/fileinfo.go | 9 +++- metainfo/info.go | 43 +++++++++++++---- metainfo/metainfo.go | 2 +- metainfo/metainfo_test.go | 13 ++++++ misc.go | 4 +- piece.go | 25 +++++++--- spec.go | 15 +++++- t.go | 9 +++- torrent.go | 86 ++++++++++++++++++++++++++-------- 17 files changed, 441 insertions(+), 55 deletions(-) create mode 100644 cmd/torrent2/main.go create mode 100644 merkle/merkle.go create mode 100644 metainfo/bep52.go diff --git a/NOTES.md b/NOTES.md index 80da84b437..45c6bbba9e 100644 --- a/NOTES.md +++ b/NOTES.md @@ -30,3 +30,9 @@ The DHT is a bit different: you can't be an active node if you are a badnat, but - https://www.bittorrent.org/beps/bep_0055.html - https://github.com/anacrolix/torrent/issues/685 - https://stackoverflow.com/questions/38786438/libutp-%C2%B5tp-and-nat-traversal-udp-hole-punching + +### BitTorrent v2 + +- https://www.bittorrent.org/beps/bep_0052.html + +The canonical infohash to use for a torrent will be the v1 infohash, or the short form of the v2 infohash if v1 is not supported. This will apply everywhere that both infohashes are present. If only one 20 byte hash is present, it is always the v1 hash (except in code that interfaces with things that only work with 20 byte hashes, like the DHT). \ No newline at end of file diff --git a/client.go b/client.go index 938bd5f306..cc472ecda7 100644 --- a/client.go +++ b/client.go @@ -43,6 +43,7 @@ import ( "github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/mse" pp "github.com/anacrolix/torrent/peer_protocol" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" request_strategy "github.com/anacrolix/torrent/request-strategy" "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/tracker" @@ -1291,8 +1292,9 @@ func (cl *Client) newTorrentOpt(opts AddTorrentOpts) (t *Torrent) { } t = &Torrent{ - cl: cl, - infoHash: opts.InfoHash, + cl: cl, + infoHash: opts.InfoHash, + infoHashV2: opts.InfoHashV2, peers: prioritizedPeers{ om: gbtree.New(32), getPrio: func(p PeerInfo) peerPriority { @@ -1396,19 +1398,21 @@ func (cl *Client) AddTorrentOpt(opts AddTorrentOpts) (t *Torrent, new bool) { } type AddTorrentOpts struct { - InfoHash infohash.T - Storage storage.ClientImpl - ChunkSize pp.Integer - InfoBytes []byte + InfoHash infohash.T + InfoHashV2 g.Option[infohash_v2.T] + Storage storage.ClientImpl + ChunkSize pp.Integer + InfoBytes []byte } // Add or merge a torrent spec. Returns new if the torrent wasn't already in the client. See also // Torrent.MergeSpec. func (cl *Client) AddTorrentSpec(spec *TorrentSpec) (t *Torrent, new bool, err error) { t, new = cl.AddTorrentOpt(AddTorrentOpts{ - InfoHash: spec.InfoHash, - Storage: spec.Storage, - ChunkSize: spec.ChunkSize, + InfoHash: spec.InfoHash, + InfoHashV2: spec.InfoHashV2, + Storage: spec.Storage, + ChunkSize: spec.ChunkSize, }) modSpec := *spec if new { @@ -1459,7 +1463,7 @@ func (t *Torrent) MergeSpec(spec *TorrentSpec) error { t.maybeNewConns() t.dataDownloadDisallowed.SetBool(spec.DisallowDataDownload) t.dataUploadDisallowed = spec.DisallowDataUpload - return nil + return t.AddPieceLayers(spec.PieceLayers) } func (cl *Client) dropTorrent(infoHash metainfo.Hash, wg *sync.WaitGroup) (err error) { diff --git a/cmd/torrent2/main.go b/cmd/torrent2/main.go new file mode 100644 index 0000000000..412e57c21d --- /dev/null +++ b/cmd/torrent2/main.go @@ -0,0 +1,44 @@ +// This is an alternate to cmd/torrent which has become bloated with awful argument parsing. Since +// this is my most complicated binary, I will try to build something that satisfies only what I need +// here. +package main + +import ( + "github.com/anacrolix/torrent/metainfo" + "os" +) + +type argError struct { + err error +} + +func assertOk(err error) { + if err != nil { + panic(err) + } +} + +func bail(str string) { + panic(str) +} + +func main() { + args := os.Args[1:] + map[string]func(){ + "metainfo": func() { + map[string]func(){ + "validate-v2": func() { + mi, err := metainfo.LoadFromFile(args[2]) + assertOk(err) + info, err := mi.UnmarshalInfo() + assertOk(err) + if !info.HasV2() { + bail("not a v2 torrent") + } + err = metainfo.ValidatePieceLayers(mi.PieceLayers, &info.FileTree, info.PieceLength) + assertOk(err) + }, + }[args[1]]() + }, + }[args[0]]() +} diff --git a/file.go b/file.go index bea4b13655..3a53adaa46 100644 --- a/file.go +++ b/file.go @@ -1,7 +1,9 @@ package torrent import ( + "crypto/sha256" "github.com/RoaringBitmap/roaring" + g "github.com/anacrolix/generics" "github.com/anacrolix/missinggo/v2/bitmap" "github.com/anacrolix/torrent/metainfo" @@ -16,6 +18,11 @@ type File struct { fi metainfo.FileInfo displayPath string prio piecePriority + piecesRoot g.Option[[sha256.Size]byte] +} + +func (f *File) String() string { + return f.Path() } func (f *File) Torrent() *Torrent { @@ -28,12 +35,12 @@ func (f *File) Offset() int64 { } // The FileInfo from the metainfo.Info to which this file corresponds. -func (f File) FileInfo() metainfo.FileInfo { +func (f *File) FileInfo() metainfo.FileInfo { return f.fi } // The file's path components joined by '/'. -func (f File) Path() string { +func (f *File) Path() string { return f.path } @@ -204,3 +211,7 @@ func (f *File) EndPieceIndex() int { } return pieceIndex((f.offset + f.length + int64(f.t.usualPieceSize()) - 1) / int64(f.t.usualPieceSize())) } + +func (f *File) numPieces() int { + return f.EndPieceIndex() - f.BeginPieceIndex() +} diff --git a/go.mod b/go.mod index 075c2bddc5..7358f281f2 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/anacrolix/torrent -go 1.21.4 - -toolchain go1.21.7 +go 1.22 require ( github.com/RoaringBitmap/roaring v1.2.3 diff --git a/merkle/merkle.go b/merkle/merkle.go new file mode 100644 index 0000000000..76985e8f78 --- /dev/null +++ b/merkle/merkle.go @@ -0,0 +1,42 @@ +package merkle + +import ( + "crypto/sha256" + "fmt" + g "github.com/anacrolix/generics" + "math/bits" +) + +func Root(hashes [][sha256.Size]byte) [sha256.Size]byte { + if len(hashes) <= 1 { + return hashes[0] + } + numHashes := uint(len(hashes)) + if numHashes != RoundUpToPowerOfTwo(uint(len(hashes))) { + panic(fmt.Sprintf("expected power of two number of hashes, got %d", numHashes)) + } + var next [][sha256.Size]byte + for i := 0; i < len(hashes); i += 2 { + left := hashes[i] + right := hashes[i+1] + h := sha256.Sum256(append(left[:], right[:]...)) + next = append(next, h) + } + return Root(next) +} + +func CompactLayerToSliceHashes(compactLayer string) (hashes [][sha256.Size]byte, err error) { + g.MakeSliceWithLength(&hashes, len(compactLayer)/sha256.Size) + for i := range hashes { + n := copy(hashes[i][:], compactLayer[i*sha256.Size:]) + if n != sha256.Size { + err = fmt.Errorf("compact layer has incomplete hash at index %d", i) + return + } + } + return +} + +func RoundUpToPowerOfTwo(n uint) (ret uint) { + return 1 << bits.Len(n-1) +} diff --git a/metainfo/bep52.go b/metainfo/bep52.go new file mode 100644 index 0000000000..18be726790 --- /dev/null +++ b/metainfo/bep52.go @@ -0,0 +1,58 @@ +package metainfo + +import ( + "fmt" + "github.com/anacrolix/torrent/merkle" +) + +func ValidatePieceLayers( + pieceLayers map[string]string, + fileTree *FileTree, + pieceLength int64, +) (err error) { + fileTree.Walk(nil, func(path []string, ft *FileTree) { + if err != nil { + return + } + if ft.IsDir() { + return + } + piecesRoot := ft.PiecesRootAsByteArray() + if !piecesRoot.Ok { + return + } + filePieceLayers, ok := pieceLayers[string(piecesRoot.Value[:])] + if !ok { + // BEP 52: "For each file in the file tree that is larger than the piece size it + // contains one string value.". The reference torrent creator in + // https://blog.libtorrent.org/2020/09/bittorrent-v2/ also has this. I'm not sure what + // harm it causes if it's present anyway, possibly it won't be useful to us. + if ft.File.Length > pieceLength { + err = fmt.Errorf("no piece layers for file %q", path) + } + return + } + var layerHashes [][32]byte + layerHashes, err = merkle.CompactLayerToSliceHashes(filePieceLayers) + padHash := HashForPiecePad(pieceLength) + for uint(len(layerHashes)) < merkle.RoundUpToPowerOfTwo(uint(len(layerHashes))) { + layerHashes = append(layerHashes, padHash) + } + var root [32]byte + root = merkle.Root(layerHashes) + if root != piecesRoot.Value { + err = fmt.Errorf("file %q: expected hash %x got %x", path, piecesRoot.Value, root) + return + } + }) + return +} + +// Returns the padding hash for the hash layer corresponding to a piece. It can't be zero because +// that's the bottom-most layer (the hashes for the smallest blocks). +func HashForPiecePad(pieceLength int64) (hash [32]byte) { + // This should be a power of two, and probably checked elsewhere. + blocksPerPiece := pieceLength / (1 << 14) + blockHashes := make([][32]byte, blocksPerPiece) + return merkle.Root(blockHashes) +} diff --git a/metainfo/file-tree.go b/metainfo/file-tree.go index 0bccb15365..3fcc4331f1 100644 --- a/metainfo/file-tree.go +++ b/metainfo/file-tree.go @@ -1,5 +1,14 @@ package metainfo +import ( + g "github.com/anacrolix/generics" + "github.com/anacrolix/torrent/bencode" + "golang.org/x/exp/maps" + "sort" +) + +const FileTreePropertiesKey = "" + type FileTree struct { File struct { Length int64 `bencode:"length"` @@ -7,3 +16,91 @@ type FileTree struct { } Dir map[string]FileTree } + +func (ft *FileTree) UnmarshalBencode(bytes []byte) (err error) { + var dir map[string]bencode.Bytes + err = bencode.Unmarshal(bytes, &dir) + if err != nil { + return + } + if propBytes, ok := dir[""]; ok { + err = bencode.Unmarshal(propBytes, &ft.File) + if err != nil { + return + } + } + delete(dir, "") + g.MakeMapWithCap(&ft.Dir, len(dir)) + for key, bytes := range dir { + var sub FileTree + err = sub.UnmarshalBencode(bytes) + if err != nil { + return + } + ft.Dir[key] = sub + } + return +} + +var _ bencode.Unmarshaler = (*FileTree)(nil) + +func (ft *FileTree) NumEntries() (num int) { + num = len(ft.Dir) + if g.MapContains(ft.Dir, FileTreePropertiesKey) { + num-- + } + return +} + +func (ft *FileTree) IsDir() bool { + return ft.NumEntries() != 0 +} + +func (ft *FileTree) orderedKeys() []string { + keys := maps.Keys(ft.Dir) + sort.Strings(keys) + return keys +} + +func (ft *FileTree) UpvertedFiles(path []string, out func(fi FileInfo)) { + if ft.IsDir() { + for _, key := range ft.orderedKeys() { + if key == FileTreePropertiesKey { + continue + } + sub := g.MapMustGet(ft.Dir, key) + sub.UpvertedFiles(append(path, key), out) + } + } else { + out(FileInfo{ + Length: ft.File.Length, + Path: append([]string(nil), path...), + // BEP 52 requires paths be UTF-8 if possible. + PathUtf8: append([]string(nil), path...), + PiecesRoot: ft.PiecesRootAsByteArray(), + }) + } +} + +func (ft *FileTree) Walk(path []string, f func(path []string, ft *FileTree)) { + f(path, ft) + for key, sub := range ft.Dir { + if key == FileTreePropertiesKey { + continue + } + sub.Walk(append(path, key), f) + } +} + +func (ft *FileTree) PiecesRootAsByteArray() (ret g.Option[[32]byte]) { + if ft.File.Length == 0 { + return + } + n := copy(ret.Value[:], ft.File.PiecesRoot) + if n != 32 { + // Must be 32 bytes for meta version 2 and non-empty files. See BEP 52. + panic(n) + } + ret.Ok = true + return +} diff --git a/metainfo/fileinfo.go b/metainfo/fileinfo.go index 894018c108..bf47215634 100644 --- a/metainfo/fileinfo.go +++ b/metainfo/fileinfo.go @@ -1,6 +1,9 @@ package metainfo -import "strings" +import ( + g "github.com/anacrolix/generics" + "strings" +) // Information specific to a single file inside the MetaInfo structure. type FileInfo struct { @@ -11,6 +14,10 @@ type FileInfo struct { PathUtf8 []string `bencode:"path.utf-8,omitempty"` ExtendedFileAttrs + + // BEP 52. This isn't encoded in a v1 FileInfo, but is exposed here for APIs that expect to deal + // v1 files. + PiecesRoot g.Option[[32]byte] `bencode:"-"` } func (fi *FileInfo) DisplayPath(info *Info) string { diff --git a/metainfo/info.go b/metainfo/info.go index b5e4d6e179..2798fd5902 100644 --- a/metainfo/info.go +++ b/metainfo/info.go @@ -125,28 +125,41 @@ func (info *Info) GeneratePieces(open func(fi FileInfo) (io.ReadCloser, error)) } func (info *Info) TotalLength() (ret int64) { - if info.IsDir() { - for _, fi := range info.Files { - ret += fi.Length - } - } else { - ret = info.Length + for _, fi := range info.UpvertedFiles() { + ret += fi.Length } return } -func (info *Info) NumPieces() int { +func (info *Info) NumPieces() (num int) { + if info.HasV2() { + info.FileTree.Walk(nil, func(path []string, ft *FileTree) { + num += int((ft.File.Length + info.PieceLength - 1) / info.PieceLength) + }) + return + } return len(info.Pieces) / 20 } +// Whether all files share the same top-level directory name. If they don't, Info.Name is usually used. func (info *Info) IsDir() bool { + if info.HasV2() { + return info.FileTree.IsDir() + } + // I wonder if we should check for the existence of Info.Length here instead. return len(info.Files) != 0 } // The files field, converted up from the old single-file in the parent info // dict if necessary. This is a helper to avoid having to conditionally handle // single and multi-file torrent infos. -func (info *Info) UpvertedFiles() []FileInfo { +func (info *Info) UpvertedFiles() (files []FileInfo) { + if info.HasV2() { + info.FileTree.UpvertedFiles(nil, func(fi FileInfo) { + files = append(files, fi) + }) + return + } if len(info.Files) == 0 { return []FileInfo{{ Length: info.Length, @@ -168,3 +181,17 @@ func (info *Info) BestName() string { } return info.Name } + +// Whether the Info can be used as a v2 info dict, including having a V2 infohash. +func (info *Info) HasV2() bool { + return info.MetaVersion == 2 +} + +func (info *Info) HasV1() bool { + // See Upgrade Path in BEP 52. + return info.MetaVersion == 0 || info.MetaVersion == 1 || info.Files != nil || info.Length != 0 || len(info.Pieces) != 0 +} + +func (info *Info) FilesArePieceAligned() bool { + return info.HasV2() +} diff --git a/metainfo/metainfo.go b/metainfo/metainfo.go index 9f4109899c..b20a9efd02 100644 --- a/metainfo/metainfo.go +++ b/metainfo/metainfo.go @@ -57,7 +57,7 @@ func (mi MetaInfo) UnmarshalInfo() (info Info, err error) { return } -func (mi MetaInfo) HashInfoBytes() (infoHash Hash) { +func (mi *MetaInfo) HashInfoBytes() (infoHash Hash) { return HashBytes(mi.InfoBytes) } diff --git a/metainfo/metainfo_test.go b/metainfo/metainfo_test.go index 335631f985..09a88e50b5 100644 --- a/metainfo/metainfo_test.go +++ b/metainfo/metainfo_test.go @@ -1,6 +1,7 @@ package metainfo import ( + "github.com/davecgh/go-spew/spew" "io" "os" "path" @@ -160,3 +161,15 @@ func TestUnmarshalEmptyStringNodes(t *testing.T) { err := bencode.Unmarshal([]byte("d5:nodes0:e"), &mi) c.Assert(err, qt.IsNil) } + +func TestUnmarshalV2Metainfo(t *testing.T) { + c := qt.New(t) + mi, err := LoadFromFile("../testdata/bittorrent-v2-test.torrent") + c.Assert(err, qt.IsNil) + info, err := mi.UnmarshalInfo() + c.Assert(err, qt.IsNil) + spew.Dump(info) + c.Check(info.NumPieces(), qt.Not(qt.Equals), 0) + err = ValidatePieceLayers(mi.PieceLayers, &info.FileTree, info.PieceLength) + c.Check(err, qt.IsNil) +} diff --git a/misc.go b/misc.go index 7d3007ecee..8f82c2a0f2 100644 --- a/misc.go +++ b/misc.go @@ -93,7 +93,9 @@ func validateInfo(info *metainfo.Info) error { if info.TotalLength() != 0 { return errors.New("zero piece length") } - } else { + } else if !info.HasV2() { + // TotalLength returns different values for V1 and V2 depending on whether v1 pad files are + // counted. Split the interface into several methods? if int((info.TotalLength()+info.PieceLength-1)/info.PieceLength) != info.NumPieces() { return errors.New("piece count and file lengths are at odds") } diff --git a/piece.go b/piece.go index 4fd2d309b9..0969653641 100644 --- a/piece.go +++ b/piece.go @@ -2,6 +2,8 @@ package torrent import ( "fmt" + g "github.com/anacrolix/generics" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "sync" "github.com/anacrolix/chansync" @@ -13,11 +15,13 @@ import ( ) type Piece struct { - // The completed piece SHA1 hash, from the metainfo "pieces" field. - hash *metainfo.Hash - t *Torrent - index pieceIndex - files []*File + // The completed piece SHA1 hash, from the metainfo "pieces" field. Nil if the info is not V1 + // compatible. + hash *metainfo.Hash + hashV2 g.Option[infohash_v2.T] + t *Torrent + index pieceIndex + files []*File readerCond chansync.BroadcastCond @@ -192,7 +196,7 @@ func (p *Piece) torrentBeginOffset() int64 { } func (p *Piece) torrentEndOffset() int64 { - return p.torrentBeginOffset() + int64(p.length()) + return p.torrentBeginOffset() + int64(p.t.usualPieceSize()) } func (p *Piece) SetPriority(prio piecePriority) { @@ -255,3 +259,12 @@ func (p *Piece) requestIndexOffset() RequestIndex { func (p *Piece) availability() int { return len(p.t.connsWithAllPieces) + p.relativeAvailability } + +// For v2 torrents, files are aligned to pieces so there should always only be a single file for a +// given piece. +func (p *Piece) mustGetOnlyFile() *File { + if len(p.files) != 1 { + panic(len(p.files)) + } + return p.files[0] +} diff --git a/spec.go b/spec.go index 8cce3cb326..f1ef584ea7 100644 --- a/spec.go +++ b/spec.go @@ -2,6 +2,8 @@ package torrent import ( "fmt" + g "github.com/anacrolix/generics" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" @@ -15,8 +17,9 @@ type TorrentSpec struct { // The tiered tracker URIs. Trackers [][]string // TODO: Move into a "new" Torrent opt type. - InfoHash metainfo.Hash - InfoBytes []byte + InfoHash metainfo.Hash + InfoHashV2 g.Option[infohash_v2.T] + InfoBytes []byte // The name to use if the Name field from the Info isn't available. DisplayName string // WebSeed URLs. For additional options add the URLs separately with Torrent.AddWebSeeds @@ -26,6 +29,8 @@ type TorrentSpec struct { PeerAddrs []string // The combination of the "xs" and "as" fields in magnet links, for now. Sources []string + // BEP 52 "piece layers" from metainfo + PieceLayers map[string]string // The chunk size to use for outbound requests. Defaults to 16KiB if not set. Can only be set // for new Torrents. TODO: Move into a "new" Torrent opt type. @@ -64,9 +69,15 @@ func TorrentSpecFromMetaInfoErr(mi *metainfo.MetaInfo) (*TorrentSpec, error) { if err != nil { err = fmt.Errorf("unmarshalling info: %w", err) } + var v2Infohash g.Option[infohash_v2.T] + if info.HasV2() { + v2Infohash.Set(infohash_v2.HashBytes(mi.InfoBytes)) + } return &TorrentSpec{ Trackers: mi.UpvertedAnnounceList(), InfoHash: mi.HashInfoBytes(), + InfoHashV2: v2Infohash, + PieceLayers: mi.PieceLayers, InfoBytes: mi.InfoBytes, DisplayName: info.Name, Webseeds: mi.UrlList, diff --git a/t.go b/t.go index 6a4607068a..83ca5a902b 100644 --- a/t.go +++ b/t.go @@ -211,19 +211,24 @@ func (t *Torrent) cancelPiecesLocked(begin, end pieceIndex, reason string) { } func (t *Torrent) initFiles() { + info := t.info var offset int64 t.files = new([]*File) for _, fi := range t.info.UpvertedFiles() { *t.files = append(*t.files, &File{ t, - strings.Join(append([]string{t.info.BestName()}, fi.BestPath()...), "/"), + strings.Join(append([]string{info.BestName()}, fi.BestPath()...), "/"), offset, fi.Length, fi, - fi.DisplayPath(t.info), + fi.DisplayPath(info), PiecePriorityNone, + fi.PiecesRoot, }) offset += fi.Length + if info.FilesArePieceAligned() { + offset = (offset + info.PieceLength - 1) / info.PieceLength * info.PieceLength + } } } diff --git a/torrent.go b/torrent.go index e27620f878..e2afb3e3f2 100644 --- a/torrent.go +++ b/torrent.go @@ -7,6 +7,8 @@ import ( "crypto/sha1" "errors" "fmt" + "github.com/anacrolix/torrent/merkle" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "io" "math/rand" "net/netip" @@ -61,10 +63,13 @@ type Torrent struct { dataUploadDisallowed bool userOnWriteChunkErr func(error) - closed chansync.SetOnce - onClose []func() - infoHash metainfo.Hash - pieces []Piece + closed chansync.SetOnce + onClose []func() + + infoHash metainfo.Hash + infoHashV2 g.Option[infohash_v2.T] + + pieces []Piece // The order pieces are requested if there's no stronger reason like availability or priority. pieceRequestOrder []int @@ -383,27 +388,59 @@ func (t *Torrent) metadataSize() int { return len(t.metadataBytes) } -func infoPieceHashes(info *metainfo.Info) (ret [][]byte) { - for i := 0; i < len(info.Pieces); i += sha1.Size { - ret = append(ret, info.Pieces[i:i+sha1.Size]) - } - return -} - func (t *Torrent) makePieces() { - hashes := infoPieceHashes(t.info) - t.pieces = make([]Piece, len(hashes)) - for i, hash := range hashes { + t.pieces = make([]Piece, t.info.NumPieces()) + for i := range t.pieces { piece := &t.pieces[i] piece.t = t - piece.index = pieceIndex(i) + piece.index = i piece.noPendingWrites.L = &piece.pendingWritesMutex - piece.hash = (*metainfo.Hash)(unsafe.Pointer(&hash[0])) + if t.info.HasV1() { + piece.hash = (*metainfo.Hash)(unsafe.Pointer( + unsafe.SliceData(t.info.Pieces[i*sha1.Size : (i+1)*sha1.Size]))) + } files := *t.files beginFile := pieceFirstFileIndex(piece.torrentBeginOffset(), files) endFile := pieceEndFileIndex(piece.torrentEndOffset(), files) piece.files = files[beginFile:endFile] + if t.info.FilesArePieceAligned() { + numFiles := len(piece.files) + if numFiles != 1 { + panic(fmt.Sprintf("%v:%v", beginFile, endFile)) + } + } + } +} + +func (t *Torrent) AddPieceLayers(layers map[string]string) (err error) { + if layers == nil { + return + } + for _, f := range *t.files { + if !f.piecesRoot.Ok { + err = fmt.Errorf("no piece root set for file %v", f) + return + } + compactLayer, ok := layers[string(f.piecesRoot.Value[:])] + if !ok { + continue + } + var hashes [][32]byte + hashes, err = merkle.CompactLayerToSliceHashes(compactLayer) + if err != nil { + err = fmt.Errorf("bad piece layers for file %q: %w", f, err) + return + } + if len(hashes) != f.numPieces() { + err = fmt.Errorf("file %q: got %v hashes expected %v", f, len(hashes), f.numPieces()) + return + } + for i := range f.numPieces() { + p := t.piece(f.BeginPieceIndex() + i) + p.hashV2.Set(hashes[i]) + } } + return nil } // Returns the index of the first file containing the piece. files must be @@ -421,11 +458,11 @@ func pieceFirstFileIndex(pieceOffset int64, files []*File) int { // ordered by offset. func pieceEndFileIndex(pieceEndOffset int64, files []*File) int { for i, f := range files { - if f.offset+f.length >= pieceEndOffset { - return i + 1 + if f.offset >= pieceEndOffset { + return i } } - return 0 + return len(files) } func (t *Torrent) cacheLength() { @@ -987,6 +1024,14 @@ func (t *Torrent) pieceLength(piece pieceIndex) pp.Integer { // There will be no variance amongst pieces. Only pain. return 0 } + if t.info.FilesArePieceAligned() { + p := t.piece(piece) + file := p.mustGetOnlyFile() + if piece == file.EndPieceIndex()-1 { + return pp.Integer(file.length - (p.torrentBeginOffset() - file.offset)) + } + return pp.Integer(t.usualPieceSize()) + } if piece == t.numPieces()-1 { ret := pp.Integer(t.length() % t.info.PieceLength) if ret != 0 { @@ -2361,6 +2406,9 @@ func (t *Torrent) peersAsSlice() (ret []*Peer) { func (t *Torrent) queuePieceCheck(pieceIndex pieceIndex) { piece := t.piece(pieceIndex) + if piece.hash == nil && !piece.hashV2.Ok { + return + } if piece.queuedForHash() { return } From 0e781be17713086f211c69005852be1918b834a4 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 27 Feb 2024 23:30:53 +1100 Subject: [PATCH 04/15] Get infohash selection working when adding v2 torrents --- spec.go | 7 ++++++- torrent.go | 9 ++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/spec.go b/spec.go index f1ef584ea7..29d30adc9f 100644 --- a/spec.go +++ b/spec.go @@ -69,13 +69,18 @@ func TorrentSpecFromMetaInfoErr(mi *metainfo.MetaInfo) (*TorrentSpec, error) { if err != nil { err = fmt.Errorf("unmarshalling info: %w", err) } + v1Ih := mi.HashInfoBytes() var v2Infohash g.Option[infohash_v2.T] if info.HasV2() { v2Infohash.Set(infohash_v2.HashBytes(mi.InfoBytes)) + if !info.HasV1() { + v1Ih = v2Infohash.Value.ToShort() + } } + return &TorrentSpec{ Trackers: mi.UpvertedAnnounceList(), - InfoHash: mi.HashInfoBytes(), + InfoHash: v1Ih, InfoHashV2: v2Infohash, PieceLayers: mi.PieceLayers, InfoBytes: mi.InfoBytes, diff --git a/torrent.go b/torrent.go index e2afb3e3f2..8416e4ba09 100644 --- a/torrent.go +++ b/torrent.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "github.com/anacrolix/torrent/merkle" + "github.com/anacrolix/torrent/types/infohash" infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "io" "math/rand" @@ -538,13 +539,19 @@ func (t *Torrent) onSetInfo() { // Called when metadata for a torrent becomes available. func (t *Torrent) setInfoBytesLocked(b []byte) error { - if metainfo.HashBytes(b) != t.infoHash { + v2Hash := infohash_v2.HashBytes(b) + v1Hash := infohash.HashBytes(b) + v2Short := v2Hash.ToShort() + if v2Short != t.infoHash && v1Hash != t.infoHash { return errors.New("info bytes have wrong hash") } var info metainfo.Info if err := bencode.Unmarshal(b, &info); err != nil { return fmt.Errorf("error unmarshalling info bytes: %s", err) } + if info.HasV2() { + t.infoHashV2.Set(v2Hash) + } t.metadataBytes = b t.metadataCompletedChunks = nil if t.info != nil { From fef859aa03e5a23cd39869f8889a1285c5e11495 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 27 Feb 2024 23:31:29 +1100 Subject: [PATCH 05/15] v2 torrent piece hashing --- go.mod | 2 +- go.sum | 4 +- merkle/hash.go | 72 ++++++++++++++++++++++++++++++++ merkle/merkle.go | 8 +++- metainfo/bep52.go | 5 ++- torrent.go | 104 ++++++++++++++++++++++++++++++++-------------- 6 files changed, 157 insertions(+), 38 deletions(-) create mode 100644 merkle/hash.go diff --git a/go.mod b/go.mod index 7358f281f2..637f205a2d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 github.com/anacrolix/envpprof v1.3.0 github.com/anacrolix/fuse v0.2.0 - github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 + github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab github.com/anacrolix/go-libutp v1.3.1 github.com/anacrolix/log v0.14.6-0.20231202035202-ed7a02cad0b4 github.com/anacrolix/missinggo v1.3.0 diff --git a/go.sum b/go.sum index d371109d1d..fb5f86b14f 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tc github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= -github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13 h1:qwOprPTDMM3BASJRf84mmZnTXRsPGGJ8xoHKQS7m3so= -github.com/anacrolix/generics v0.0.0-20230911070922-5dd7545c6b13/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab h1:MvuAC/UJtcohN6xWc8zYXSZfllh1LVNepQ0R3BCX5I4= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0= github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= diff --git a/merkle/hash.go b/merkle/hash.go new file mode 100644 index 0000000000..18ecee78e0 --- /dev/null +++ b/merkle/hash.go @@ -0,0 +1,72 @@ +package merkle + +import ( + "crypto/sha256" + "hash" +) + +func NewHash() *Hash { + return &Hash{ + nextBlock: sha256.New(), + } +} + +type Hash struct { + blocks [][32]byte + nextBlock hash.Hash + written int +} + +func (h *Hash) remaining() int { + return BlockSize - h.written +} + +func (h *Hash) Write(p []byte) (n int, err error) { + for len(p) > 0 { + var n1 int + n1, err = h.nextBlock.Write(p[:min(len(p), h.remaining())]) + n += n1 + h.written += n1 + p = p[n1:] + if h.remaining() == 0 { + h.blocks = append(h.blocks, h.nextBlockSum()) + h.nextBlock.Reset() + h.written = 0 + } + if err != nil { + break + } + } + return +} + +func (h *Hash) nextBlockSum() (sum [32]byte) { + h.nextBlock.Sum(sum[:0]) + return +} + +func (h *Hash) Sum(b []byte) []byte { + blocks := h.blocks + if h.written != 0 { + blocks = append(blocks, h.nextBlockSum()) + } + n := int(RoundUpToPowerOfTwo(uint(len(blocks)))) + blocks = append(blocks, make([][32]byte, n-len(blocks))...) + sum := Root(blocks) + return append(b, sum[:]...) +} + +func (h *Hash) Reset() { + h.blocks = h.blocks[:0] + h.nextBlock.Reset() +} + +func (h *Hash) Size() int { + return 32 +} + +func (h *Hash) BlockSize() int { + return h.nextBlock.BlockSize() +} + +var _ hash.Hash = (*Hash)(nil) diff --git a/merkle/merkle.go b/merkle/merkle.go index 76985e8f78..a6667cb42b 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -7,8 +7,14 @@ import ( "math/bits" ) +// The leaf block size for BitTorrent v2 Merkle trees. +const BlockSize = 1 << 14 // 16KiB + func Root(hashes [][sha256.Size]byte) [sha256.Size]byte { - if len(hashes) <= 1 { + switch len(hashes) { + case 0: + return sha256.Sum256(nil) + case 1: return hashes[0] } numHashes := uint(len(hashes)) diff --git a/metainfo/bep52.go b/metainfo/bep52.go index 18be726790..8bdd19deb3 100644 --- a/metainfo/bep52.go +++ b/metainfo/bep52.go @@ -25,8 +25,9 @@ func ValidatePieceLayers( if !ok { // BEP 52: "For each file in the file tree that is larger than the piece size it // contains one string value.". The reference torrent creator in - // https://blog.libtorrent.org/2020/09/bittorrent-v2/ also has this. I'm not sure what - // harm it causes if it's present anyway, possibly it won't be useful to us. + // https://blog.libtorrent.org/2020/09/bittorrent-v2/ also has this. If a file is equal + // to or smaller than the piece length, we can just use the pieces root instead of the + // piece layer hash. if ft.File.Length > pieceLength { err = fmt.Errorf("no piece layers for file %q", path) } diff --git a/torrent.go b/torrent.go index 8416e4ba09..f15459b5af 100644 --- a/torrent.go +++ b/torrent.go @@ -10,6 +10,7 @@ import ( "github.com/anacrolix/torrent/merkle" "github.com/anacrolix/torrent/types/infohash" infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" + "hash" "io" "math/rand" "net/netip" @@ -423,22 +424,35 @@ func (t *Torrent) AddPieceLayers(layers map[string]string) (err error) { return } compactLayer, ok := layers[string(f.piecesRoot.Value[:])] - if !ok { - continue - } var hashes [][32]byte - hashes, err = merkle.CompactLayerToSliceHashes(compactLayer) - if err != nil { - err = fmt.Errorf("bad piece layers for file %q: %w", f, err) - return + if ok { + hashes, err = merkle.CompactLayerToSliceHashes(compactLayer) + if err != nil { + err = fmt.Errorf("bad piece layers for file %q: %w", f, err) + return + } + } else if f.length > t.info.PieceLength { + // BEP 52 is pretty strongly worded about this, even though we should be able to + // recover: If a v2 torrent is added by magnet link or infohash, we need to fetch piece + // layers ourselves anyway, and that's how we can recover from this. + t.logger.Levelf(log.Warning, "no piece layers for file %q", f) + continue + } else { + hashes = [][32]byte{f.piecesRoot.Value} } if len(hashes) != f.numPieces() { err = fmt.Errorf("file %q: got %v hashes expected %v", f, len(hashes), f.numPieces()) return } for i := range f.numPieces() { - p := t.piece(f.BeginPieceIndex() + i) - p.hashV2.Set(hashes[i]) + pi := f.BeginPieceIndex() + i + p := t.piece(pi) + // See Torrent.onSetInfo. We want to trigger an initial check if appropriate, if we + // didn't yet have a piece hash (can occur with v2 when we don't start with piece + // layers). + if !p.hashV2.Set(hashes[i]).Ok && p.hash == nil { + t.queueInitialPieceCheck(pi) + } } } return nil @@ -521,10 +535,7 @@ func (t *Torrent) onSetInfo() { p.relativeAvailability = t.selectivePieceAvailabilityFromPeers(i) t.addRequestOrderPiece(i) t.updatePieceCompletion(i) - if !t.initialPieceCheckDisabled && !p.storageCompletionOk { - // t.logger.Printf("piece %s completion unknown, queueing check", p) - t.queuePieceCheck(i) - } + t.queueInitialPieceCheck(i) } t.cl.event.Broadcast() close(t.gotMetainfoC) @@ -1057,28 +1068,39 @@ func (t *Torrent) smartBanBlockCheckingWriter(piece pieceIndex) *blockCheckingWr } func (t *Torrent) hashPiece(piece pieceIndex) ( - ret metainfo.Hash, + correct bool, // These are peers that sent us blocks that differ from what we hash here. differingPeers map[bannableAddr]struct{}, err error, ) { p := t.piece(piece) p.waitNoPendingWrites() - storagePiece := t.pieces[piece].Storage() - - // Does the backend want to do its own hashing? - if i, ok := storagePiece.PieceImpl.(storage.SelfHashing); ok { - var sum metainfo.Hash - // log.Printf("A piece decided to self-hash: %d", piece) - sum, err = i.SelfHash() - missinggo.CopyExact(&ret, sum) - return + storagePiece := p.Storage() + + var h hash.Hash + if p.hash != nil { + h = pieceHash.New() + + // Does the backend want to do its own hashing? + if i, ok := storagePiece.PieceImpl.(storage.SelfHashing); ok { + var sum metainfo.Hash + // log.Printf("A piece decided to self-hash: %d", piece) + sum, err = i.SelfHash() + correct = sum == *p.hash + // Can't do smart banning without reading the piece. The smartBanCache is still cleared + // in pieceHasher regardless. + return + } + + } else if p.hashV2.Ok { + h = merkle.NewHash() + } else { + panic("no hash") } - hash := pieceHash.New() const logPieceContents = false smartBanWriter := t.smartBanBlockCheckingWriter(piece) - writers := []io.Writer{hash, smartBanWriter} + writers := []io.Writer{h, smartBanWriter} var examineBuf bytes.Buffer if logPieceContents { writers = append(writers, &examineBuf) @@ -1089,7 +1111,23 @@ func (t *Torrent) hashPiece(piece pieceIndex) ( } smartBanWriter.Flush() differingPeers = smartBanWriter.badPeers - missinggo.CopyExact(&ret, hash.Sum(nil)) + if p.hash != nil { + var sum [20]byte + n := len(h.Sum(sum[:0])) + if n != 20 { + panic(n) + } + correct = sum == *p.hash + } else if p.hashV2.Ok { + var sum [32]byte + n := len(h.Sum(sum[:0])) + if n != 32 { + panic(n) + } + correct = sum == p.hashV2.Value + } else { + panic("no hash") + } return } @@ -2169,10 +2207,7 @@ func (t *Torrent) pieceHashed(piece pieceIndex, passed bool, hashIoErr error) { } else { log.Fmsg( "piece %d failed hash: %d connections contributed", piece, len(p.dirtiers), - ).AddValues(t, p).LogLevel( - - log.Debug, t.logger) - + ).AddValues(t, p).LogLevel(log.Info, t.logger) pieceHashedNotCorrect.Add(1) } } @@ -2368,8 +2403,7 @@ func (t *Torrent) dropBannedPeers() { func (t *Torrent) pieceHasher(index pieceIndex) { p := t.piece(index) - sum, failedPeers, copyErr := t.hashPiece(index) - correct := sum == *p.hash + correct, failedPeers, copyErr := t.hashPiece(index) switch copyErr { case nil, io.EOF: default: @@ -2411,6 +2445,12 @@ func (t *Torrent) peersAsSlice() (ret []*Peer) { return } +func (t *Torrent) queueInitialPieceCheck(i pieceIndex) { + if !t.initialPieceCheckDisabled && !t.piece(i).storageCompletionOk { + t.queuePieceCheck(i) + } +} + func (t *Torrent) queuePieceCheck(pieceIndex pieceIndex) { piece := t.piece(pieceIndex) if piece.hash == nil && !piece.hashV2.Ok { From 1fd47d6748161c5c915c24fd7528e9d4add24319 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Tue, 27 Feb 2024 23:31:43 +1100 Subject: [PATCH 06/15] Warn on unhandled v2 protocol messages --- peer_protocol/protocol.go | 5 +++++ peerconn.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/peer_protocol/protocol.go b/peer_protocol/protocol.go index bfeb6a0448..2f92ab324d 100644 --- a/peer_protocol/protocol.go +++ b/peer_protocol/protocol.go @@ -41,6 +41,11 @@ const ( // BEP 10 Extended MessageType = 20 + + // BEP 52 + HashRequest = 21 + Hashes = 22 + HashReject = 23 ) const ( diff --git a/peerconn.go b/peerconn.go index a1e98b1282..ded9b483e5 100644 --- a/peerconn.go +++ b/peerconn.go @@ -862,6 +862,8 @@ func (c *PeerConn) mainReadLoop() (err error) { c.updateRequests("PeerConn.mainReadLoop allowed fast") case pp.Extended: err = c.onReadExtendedMsg(msg.ExtendedID, msg.ExtendedPayload) + case pp.HashRequest, pp.Hashes, pp.HashReject: + err = log.WithLevel(log.Warning, fmt.Errorf("received unimplemented BitTorrent v2 message: %v", msg.Type)) default: err = fmt.Errorf("received unknown message type: %#v", msg.Type) } From 13b339cf750351316d436d0f7918620177156772 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 28 Feb 2024 00:15:40 +1100 Subject: [PATCH 07/15] Fix metainfo.Piece.Length for v2 torrents --- metainfo/fileinfo.go | 12 +----------- metainfo/info.go | 2 +- metainfo/piece.go | 27 +++++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/metainfo/fileinfo.go b/metainfo/fileinfo.go index bf47215634..82e1c94f81 100644 --- a/metainfo/fileinfo.go +++ b/metainfo/fileinfo.go @@ -28,17 +28,7 @@ func (fi *FileInfo) DisplayPath(info *Info) string { } } -func (me FileInfo) Offset(info *Info) (ret int64) { - for _, fi := range info.UpvertedFiles() { - if me.DisplayPath(info) == fi.DisplayPath(info) { - return - } - ret += fi.Length - } - panic("not found") -} - -func (fi FileInfo) BestPath() []string { +func (fi *FileInfo) BestPath() []string { if len(fi.PathUtf8) != 0 { return fi.PathUtf8 } diff --git a/metainfo/info.go b/metainfo/info.go index 2798fd5902..d1b54d44d6 100644 --- a/metainfo/info.go +++ b/metainfo/info.go @@ -172,7 +172,7 @@ func (info *Info) UpvertedFiles() (files []FileInfo) { } func (info *Info) Piece(index int) Piece { - return Piece{info, pieceIndex(index)} + return Piece{info, index} } func (info *Info) BestName() string { diff --git a/metainfo/piece.go b/metainfo/piece.go index d8895384d0..c7377f5db6 100644 --- a/metainfo/piece.go +++ b/metainfo/piece.go @@ -8,7 +8,30 @@ type Piece struct { type pieceIndex = int func (p Piece) Length() int64 { - if int(p.i) == p.Info.NumPieces()-1 { + if p.Info.HasV2() { + var offset int64 + pieceLength := p.Info.PieceLength + lastFileEnd := int64(0) + done := false + p.Info.FileTree.UpvertedFiles(nil, func(fi FileInfo) { + if done { + return + } + fileStartPiece := int(offset / pieceLength) + if fileStartPiece > p.i { + done = true + return + } + lastFileEnd = offset + fi.Length + offset = (lastFileEnd + pieceLength - 1) / pieceLength * pieceLength + }) + ret := min(lastFileEnd-int64(p.i)*pieceLength, pieceLength) + if ret <= 0 { + panic(ret) + } + return ret + } + if p.i == p.Info.NumPieces()-1 { return p.Info.TotalLength() - int64(p.i)*p.Info.PieceLength } return p.Info.PieceLength @@ -23,6 +46,6 @@ func (p Piece) Hash() (ret Hash) { return } -func (p Piece) Index() pieceIndex { +func (p Piece) Index() int { return p.i } From 80b1560de3a47bc139adb06aa5f7510ceccfe49b Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Wed, 28 Feb 2024 19:19:16 +1100 Subject: [PATCH 08/15] Announce to both v1 and v2 swarms --- client.go | 46 ++++++--- client_test.go | 12 ++- cmd/torrent2/main.go | 3 +- file.go | 1 + issue97_test.go | 2 + ltep_test.go | 2 + merkle/merkle.go | 3 +- metainfo/bep52.go | 1 + metainfo/file-tree.go | 6 +- metainfo/fileinfo.go | 3 +- metainfo/magnet-v2.go | 2 +- metainfo/magnet_test.go | 4 +- metainfo/metainfo_test.go | 2 +- peerconn.go | 3 + peerconn_test.go | 4 +- pexconn_test.go | 3 +- piece.go | 6 +- reader.go | 4 +- requesting.go | 2 +- spec.go | 10 +- t.go | 6 +- test_test.go | 2 +- torrent.go | 163 ++++++++++++++++++++++++------- torrent_test.go | 6 +- tracker_scraper.go | 8 +- types/infohash-v2/infohash-v2.go | 6 +- types/infohash/infohash.go | 4 + 27 files changed, 227 insertions(+), 87 deletions(-) diff --git a/client.go b/client.go index cc472ecda7..4b8e9df1ff 100644 --- a/client.go +++ b/client.go @@ -43,11 +43,11 @@ import ( "github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/mse" pp "github.com/anacrolix/torrent/peer_protocol" - infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" request_strategy "github.com/anacrolix/torrent/request-strategy" "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/tracker" "github.com/anacrolix/torrent/types/infohash" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "github.com/anacrolix/torrent/webtorrent" ) @@ -156,7 +156,7 @@ func (cl *Client) WriteStatus(_w io.Writer) { fmt.Fprintf(w, "# Torrents: %d\n", len(torrentsSlice)) fmt.Fprintln(w) sort.Slice(torrentsSlice, func(l, r int) bool { - return torrentsSlice[l].infoHash.AsString() < torrentsSlice[r].infoHash.AsString() + return torrentsSlice[l].canonicalShortInfohash().AsString() < torrentsSlice[r].canonicalShortInfohash().AsString() }) for _, t := range torrentsSlice { if t.name() == "" { @@ -306,14 +306,18 @@ func NewClient(cfg *ClientConfig) (cl *Client, err error) { cl.websocketTrackers = websocketTrackers{ PeerId: cl.peerID, Logger: cl.logger, - GetAnnounceRequest: func(event tracker.AnnounceEvent, infoHash [20]byte) (tracker.AnnounceRequest, error) { + GetAnnounceRequest: func( + event tracker.AnnounceEvent, infoHash [20]byte, + ) ( + tracker.AnnounceRequest, error, + ) { cl.lock() defer cl.unlock() t, ok := cl.torrents[infoHash] if !ok { return tracker.AnnounceRequest{}, errors.New("torrent not tracked by client") } - return t.announceRequest(event), nil + return t.announceRequest(event, infoHash), nil }, Proxy: cl.config.HTTPProxy, WebsocketTrackerHttpHeader: cl.config.WebsocketTrackerHttpHeader, @@ -903,16 +907,15 @@ func (cl *Client) incomingPeerPort() int { return cl.LocalPort() } -func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) error { +func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) (err error) { if c.headerEncrypted { var rw io.ReadWriter - var err error rw, c.cryptoMethod, err = mse.InitiateHandshake( struct { io.Reader io.Writer }{c.r, c.w}, - t.infoHash[:], + t.canonicalShortInfohash().Bytes(), nil, cl.config.CryptoProvides, ) @@ -921,14 +924,19 @@ func (cl *Client) initiateHandshakes(c *PeerConn, t *Torrent) error { return fmt.Errorf("header obfuscation handshake: %w", err) } } - ih, err := cl.connBtHandshake(c, &t.infoHash) + ih, err := cl.connBtHandshake(c, t.canonicalShortInfohash()) if err != nil { return fmt.Errorf("bittorrent protocol handshake: %w", err) } - if ih != t.infoHash { - return errors.New("bittorrent protocol handshake: peer infohash didn't match") + if g.Some(ih) == t.infoHash { + return nil } - return nil + if t.infoHashV2.Ok && *t.infoHashV2.Value.ToShort() == ih { + c.v2 = true + return nil + } + err = errors.New("bittorrent protocol handshake: peer infohash didn't match") + return } // Calls f with any secret keys. Note that it takes the Client lock, and so must be used from code @@ -1285,6 +1293,13 @@ func (cl *Client) newTorrent(ih metainfo.Hash, specStorage storage.ClientImpl) ( // Return a Torrent ready for insertion into a Client. func (cl *Client) newTorrentOpt(opts AddTorrentOpts) (t *Torrent) { + var v1InfoHash g.Option[infohash.T] + if !opts.InfoHash.IsZero() { + v1InfoHash.Set(opts.InfoHash) + } + if !v1InfoHash.Ok && !opts.InfoHashV2.Ok { + panic("v1 infohash must be nonzero or v2 infohash must be set") + } // use provided storage, if provided storageClient := cl.defaultStorage if opts.Storage != nil { @@ -1293,7 +1308,7 @@ func (cl *Client) newTorrentOpt(opts AddTorrentOpts) (t *Torrent) { t = &Torrent{ cl: cl, - infoHash: opts.InfoHash, + infoHash: v1InfoHash, infoHashV2: opts.InfoHashV2, peers: prioritizedPeers{ om: gbtree.New(32), @@ -1344,10 +1359,13 @@ func (cl *Client) AddTorrentInfoHash(infoHash metainfo.Hash) (t *Torrent, new bo return cl.AddTorrentInfoHashWithStorage(infoHash, nil) } -// Adds a torrent by InfoHash with a custom Storage implementation. +// Deprecated. Adds a torrent by InfoHash with a custom Storage implementation. // If the torrent already exists then this Storage is ignored and the // existing torrent returned with `new` set to `false` -func (cl *Client) AddTorrentInfoHashWithStorage(infoHash metainfo.Hash, specStorage storage.ClientImpl) (t *Torrent, new bool) { +func (cl *Client) AddTorrentInfoHashWithStorage( + infoHash metainfo.Hash, + specStorage storage.ClientImpl, +) (t *Torrent, new bool) { cl.lock() defer cl.unlock() t, ok := cl.torrents[infoHash] diff --git a/client_test.go b/client_test.go index 5463d41c3f..781f04a914 100644 --- a/client_test.go +++ b/client_test.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "io" + "math/rand" "net" "net/netip" "os" @@ -131,9 +132,9 @@ func TestAddDropManyTorrents(t *testing.T) { cl, err := NewClient(TestingConfig(t)) require.NoError(t, err) defer cl.Close() - for i := 0; i < 1000; i += 1 { + for i := range 1000 { var spec TorrentSpec - binary.PutVarint(spec.InfoHash[:], int64(i)) + binary.PutVarint(spec.InfoHash[:], int64(i+1)) tt, new, err := cl.AddTorrentSpec(&spec) assert.NoError(t, err) assert.True(t, new) @@ -155,6 +156,7 @@ func TestMergingTrackersByAddingSpecs(t *testing.T) { require.NoError(t, err) defer cl.Close() spec := TorrentSpec{} + rand.Read(spec.InfoHash[:]) T, new, _ := cl.AddTorrentSpec(&spec) if !new { t.FailNow() @@ -587,6 +589,7 @@ func TestPeerInvalidHave(t *testing.T) { } func TestPieceCompletedInStorageButNotClient(t *testing.T) { + c := qt.New(t) greetingTempDir, greetingMetainfo := testutil.GreetingTestTorrent() defer os.RemoveAll(greetingTempDir) cfg := TestingConfig(t) @@ -594,9 +597,12 @@ func TestPieceCompletedInStorageButNotClient(t *testing.T) { seeder, err := NewClient(TestingConfig(t)) require.NoError(t, err) defer seeder.Close() - seeder.AddTorrentSpec(&TorrentSpec{ + _, new, err := seeder.AddTorrentSpec(&TorrentSpec{ InfoBytes: greetingMetainfo.InfoBytes, + InfoHash: greetingMetainfo.HashInfoBytes(), }) + c.Check(err, qt.IsNil) + c.Check(new, qt.IsTrue) } // Check that when the listen port is 0, all the protocols listened on have diff --git a/cmd/torrent2/main.go b/cmd/torrent2/main.go index 412e57c21d..0d57f0133b 100644 --- a/cmd/torrent2/main.go +++ b/cmd/torrent2/main.go @@ -4,8 +4,9 @@ package main import ( - "github.com/anacrolix/torrent/metainfo" "os" + + "github.com/anacrolix/torrent/metainfo" ) type argError struct { diff --git a/file.go b/file.go index 3a53adaa46..ae802f132c 100644 --- a/file.go +++ b/file.go @@ -2,6 +2,7 @@ package torrent import ( "crypto/sha256" + "github.com/RoaringBitmap/roaring" g "github.com/anacrolix/generics" "github.com/anacrolix/missinggo/v2/bitmap" diff --git a/issue97_test.go b/issue97_test.go index ee8107c6dd..9e14f047c0 100644 --- a/issue97_test.go +++ b/issue97_test.go @@ -19,6 +19,8 @@ func TestHashPieceAfterStorageClosed(t *testing.T) { logger: log.Default, chunkSize: defaultChunkSize, } + tt.infoHash.Ok = true + tt.infoHash.Value[0] = 1 mi := testutil.GreetingMetaInfo() info, err := mi.UnmarshalInfo() require.NoError(t, err) diff --git a/ltep_test.go b/ltep_test.go index 0a95a86837..37300f09cf 100644 --- a/ltep_test.go +++ b/ltep_test.go @@ -1,6 +1,7 @@ package torrent_test import ( + "math/rand" "strconv" "testing" @@ -113,6 +114,7 @@ func TestUserLtep(t *testing.T) { c.Assert(err, qt.IsNil) defer cl2.Close() addOpts := AddTorrentOpts{} + rand.Read(addOpts.InfoHash[:]) t1, _ := cl1.AddTorrentOpt(addOpts) t2, _ := cl2.AddTorrentOpt(addOpts) defer testutil.ExportStatusWriter(cl1, "cl1", t)() diff --git a/merkle/merkle.go b/merkle/merkle.go index a6667cb42b..ab54af6a2b 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -3,8 +3,9 @@ package merkle import ( "crypto/sha256" "fmt" - g "github.com/anacrolix/generics" "math/bits" + + g "github.com/anacrolix/generics" ) // The leaf block size for BitTorrent v2 Merkle trees. diff --git a/metainfo/bep52.go b/metainfo/bep52.go index 8bdd19deb3..0291d65349 100644 --- a/metainfo/bep52.go +++ b/metainfo/bep52.go @@ -2,6 +2,7 @@ package metainfo import ( "fmt" + "github.com/anacrolix/torrent/merkle" ) diff --git a/metainfo/file-tree.go b/metainfo/file-tree.go index 3fcc4331f1..bfb7229e9a 100644 --- a/metainfo/file-tree.go +++ b/metainfo/file-tree.go @@ -1,10 +1,12 @@ package metainfo import ( + "sort" + g "github.com/anacrolix/generics" - "github.com/anacrolix/torrent/bencode" "golang.org/x/exp/maps" - "sort" + + "github.com/anacrolix/torrent/bencode" ) const FileTreePropertiesKey = "" diff --git a/metainfo/fileinfo.go b/metainfo/fileinfo.go index 82e1c94f81..66ee2da8b8 100644 --- a/metainfo/fileinfo.go +++ b/metainfo/fileinfo.go @@ -1,8 +1,9 @@ package metainfo import ( - g "github.com/anacrolix/generics" "strings" + + g "github.com/anacrolix/generics" ) // Information specific to a single file inside the MetaInfo structure. diff --git a/metainfo/magnet-v2.go b/metainfo/magnet-v2.go index a6c2c8b4d4..322c624d42 100644 --- a/metainfo/magnet-v2.go +++ b/metainfo/magnet-v2.go @@ -7,9 +7,9 @@ import ( "net/url" "strings" + g "github.com/anacrolix/generics" "github.com/multiformats/go-multihash" - g "github.com/anacrolix/generics" infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" ) diff --git a/metainfo/magnet_test.go b/metainfo/magnet_test.go index ba13ac8857..2547509894 100644 --- a/metainfo/magnet_test.go +++ b/metainfo/magnet_test.go @@ -2,10 +2,10 @@ package metainfo import ( "encoding/hex" - "github.com/davecgh/go-spew/spew" - qt "github.com/frankban/quicktest" "testing" + "github.com/davecgh/go-spew/spew" + qt "github.com/frankban/quicktest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/metainfo/metainfo_test.go b/metainfo/metainfo_test.go index 09a88e50b5..9f0f9f5f73 100644 --- a/metainfo/metainfo_test.go +++ b/metainfo/metainfo_test.go @@ -1,7 +1,6 @@ package metainfo import ( - "github.com/davecgh/go-spew/spew" "io" "os" "path" @@ -10,6 +9,7 @@ import ( "testing" "github.com/anacrolix/missinggo/v2" + "github.com/davecgh/go-spew/spew" qt "github.com/frankban/quicktest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/peerconn.go b/peerconn.go index ded9b483e5..59a31ad4a7 100644 --- a/peerconn.go +++ b/peerconn.go @@ -36,6 +36,9 @@ import ( type PeerConn struct { Peer + // BEP 52 + v2 bool + // A string that should identify the PeerConn's net.Conn endpoints. The net.Conn could // be wrapping WebRTC, uTP, or TCP etc. Used in writing the conn status for peers. connString string diff --git a/peerconn_test.go b/peerconn_test.go index 70f9017bb0..7dc28bd1ac 100644 --- a/peerconn_test.go +++ b/peerconn_test.go @@ -29,7 +29,7 @@ func TestSendBitfieldThenHave(t *testing.T) { cl.initLogger() qtc := qt.New(t) c := cl.newConnection(nil, newConnectionOpts{network: "io.Pipe"}) - c.setTorrent(cl.newTorrent(metainfo.Hash{}, nil)) + c.setTorrent(cl.newTorrentForTesting()) err := c.t.setInfo(&metainfo.Info{Pieces: make([]byte, metainfo.HashSize*3)}) qtc.Assert(err, qt.IsNil) r, w := io.Pipe() @@ -98,7 +98,7 @@ func BenchmarkConnectionMainReadLoop(b *testing.B) { }) cl.initLogger() ts := &torrentStorage{} - t := cl.newTorrent(metainfo.Hash{}, nil) + t := cl.newTorrentForTesting() t.initialPieceCheckDisabled = true require.NoError(b, t.setInfo(&metainfo.Info{ Pieces: make([]byte, 20), diff --git a/pexconn_test.go b/pexconn_test.go index b8be73e887..02da7d0a1b 100644 --- a/pexconn_test.go +++ b/pexconn_test.go @@ -7,7 +7,6 @@ import ( "github.com/anacrolix/dht/v2/krpc" "github.com/stretchr/testify/require" - "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" ) @@ -15,7 +14,7 @@ func TestPexConnState(t *testing.T) { var cl Client cl.init(TestingConfig(t)) cl.initLogger() - torrent := cl.newTorrent(metainfo.Hash{}, nil) + torrent := cl.newTorrentForTesting() addr := &net.TCPAddr{IP: net.IPv6loopback, Port: 4747} c := cl.newConnection(nil, newConnectionOpts{ remoteAddr: addr, diff --git a/piece.go b/piece.go index 0969653641..607e2f883f 100644 --- a/piece.go +++ b/piece.go @@ -2,16 +2,16 @@ package torrent import ( "fmt" - g "github.com/anacrolix/generics" - infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "sync" "github.com/anacrolix/chansync" + g "github.com/anacrolix/generics" "github.com/anacrolix/missinggo/v2/bitmap" "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" "github.com/anacrolix/torrent/storage" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" ) type Piece struct { @@ -48,7 +48,7 @@ type Piece struct { } func (p *Piece) String() string { - return fmt.Sprintf("%s/%d", p.t.infoHash.HexString(), p.index) + return fmt.Sprintf("%s/%d", p.t.canonicalShortInfohash().HexString(), p.index) } func (p *Piece) Info() metainfo.Piece { diff --git a/reader.go b/reader.go index 4b20206cbf..039fd99cc9 100644 --- a/reader.go +++ b/reader.go @@ -261,8 +261,8 @@ func (r *reader) readOnceAt(ctx context.Context, b []byte, pos int64) (n int, er } // TODO: Just reset pieces in the readahead window. This might help // prevent thrashing with small caches and file and piece priorities. - r.log(log.Fstr("error reading torrent %s piece %d offset %d, %d bytes: %v", - r.t.infoHash.HexString(), firstPieceIndex, firstPieceOffset, len(b1), err)) + r.log(log.Fstr("error reading piece %d offset %d, %d bytes: %v", + firstPieceIndex, firstPieceOffset, len(b1), err)) if !r.t.updatePieceCompletion(firstPieceIndex) { r.log(log.Fstr("piece %d completion unchanged", firstPieceIndex)) } diff --git a/requesting.go b/requesting.go index 1440207d52..b48fc79f50 100644 --- a/requesting.go +++ b/requesting.go @@ -189,7 +189,7 @@ func (p *Peer) getDesiredRequestState() (desired desiredRequestState) { input, t.getPieceRequestOrder(), func(ih InfoHash, pieceIndex int, pieceExtra requestStrategy.PieceRequestOrderState) { - if ih != t.infoHash { + if ih != *t.canonicalShortInfohash() { return } if !p.peerHasPiece(pieceIndex) { diff --git a/spec.go b/spec.go index 29d30adc9f..49a1104ede 100644 --- a/spec.go +++ b/spec.go @@ -2,12 +2,13 @@ package torrent import ( "fmt" + g "github.com/anacrolix/generics" - infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" "github.com/anacrolix/torrent/storage" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" ) // Specifies a new torrent for adding to a client, or additions to an existing Torrent. There are @@ -46,14 +47,15 @@ type TorrentSpec struct { } func TorrentSpecFromMagnetUri(uri string) (spec *TorrentSpec, err error) { - m, err := metainfo.ParseMagnetUri(uri) + m, err := metainfo.ParseMagnetV2Uri(uri) if err != nil { return } spec = &TorrentSpec{ Trackers: [][]string{m.Trackers}, DisplayName: m.DisplayName, - InfoHash: m.InfoHash, + InfoHash: m.InfoHash.UnwrapOrZeroValue(), + InfoHashV2: m.V2InfoHash, Webseeds: m.Params["ws"], Sources: append(m.Params["xs"], m.Params["as"]...), PeerAddrs: m.Params["x.pe"], // BEP 9 @@ -74,7 +76,7 @@ func TorrentSpecFromMetaInfoErr(mi *metainfo.MetaInfo) (*TorrentSpec, error) { if info.HasV2() { v2Infohash.Set(infohash_v2.HashBytes(mi.InfoBytes)) if !info.HasV1() { - v1Ih = v2Infohash.Value.ToShort() + v1Ih = *v2Infohash.Value.ToShort() } } diff --git a/t.go b/t.go index 83ca5a902b..dda65d363e 100644 --- a/t.go +++ b/t.go @@ -13,7 +13,7 @@ import ( // The Torrent's infohash. This is fixed and cannot change. It uniquely identifies a torrent. func (t *Torrent) InfoHash() metainfo.Hash { - return t.infoHash + return *t.canonicalShortInfohash() } // Returns a channel that is closed when the info (.Info()) for the torrent has become available. @@ -100,7 +100,7 @@ func (t *Torrent) Drop() { defer wg.Wait() t.cl.lock() defer t.cl.unlock() - err := t.cl.dropTorrent(t.infoHash, &wg) + err := t.cl.dropTorrent(*t.canonicalShortInfohash(), &wg) if err != nil { panic(err) } @@ -254,7 +254,7 @@ func (t *Torrent) DownloadAll() { func (t *Torrent) String() string { s := t.name() if s == "" { - return t.infoHash.HexString() + return t.canonicalShortInfohash().HexString() } else { return strconv.Quote(s) } diff --git a/test_test.go b/test_test.go index 6babc911c3..11ebb3b3d1 100644 --- a/test_test.go +++ b/test_test.go @@ -19,5 +19,5 @@ func newTestingClient(t testing.TB) *Client { } func (cl *Client) newTorrentForTesting() *Torrent { - return cl.newTorrent(metainfo.Hash{}, nil) + return cl.newTorrent(metainfo.Hash{1}, nil) } diff --git a/torrent.go b/torrent.go index f15459b5af..3c925ae577 100644 --- a/torrent.go +++ b/torrent.go @@ -7,9 +7,6 @@ import ( "crypto/sha1" "errors" "fmt" - "github.com/anacrolix/torrent/merkle" - "github.com/anacrolix/torrent/types/infohash" - infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "hash" "io" "math/rand" @@ -36,10 +33,12 @@ import ( "github.com/anacrolix/sync" "github.com/pion/datachannel" "golang.org/x/exp/maps" + "golang.org/x/sync/errgroup" "github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/internal/check" "github.com/anacrolix/torrent/internal/nestedmaps" + "github.com/anacrolix/torrent/merkle" "github.com/anacrolix/torrent/metainfo" pp "github.com/anacrolix/torrent/peer_protocol" utHolepunch "github.com/anacrolix/torrent/peer_protocol/ut-holepunch" @@ -47,6 +46,8 @@ import ( "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/tracker" typedRoaring "github.com/anacrolix/torrent/typed-roaring" + "github.com/anacrolix/torrent/types/infohash" + infohash_v2 "github.com/anacrolix/torrent/types/infohash-v2" "github.com/anacrolix/torrent/webseed" "github.com/anacrolix/torrent/webtorrent" ) @@ -68,7 +69,7 @@ type Torrent struct { closed chansync.SetOnce onClose []func() - infoHash metainfo.Hash + infoHash g.Option[metainfo.Hash] infoHashV2 g.Option[infohash_v2.T] pieces []Piece @@ -118,7 +119,7 @@ type Torrent struct { // Whether we want to know more peers. wantPeersEvent missinggo.Event // An announcer for each tracker URL. - trackerAnnouncers map[string]torrentTrackerAnnouncer + trackerAnnouncers map[torrentTrackerAnnouncerKey]torrentTrackerAnnouncer // How many times we've initiated a DHT announce. TODO: Move into stats. numDHTAnnounces int @@ -177,6 +178,11 @@ type Torrent struct { disableTriggers bool } +type torrentTrackerAnnouncerKey struct { + shortInfohash [20]byte + url string +} + type outgoingConnAttemptKey = *PeerInfo func (t *Torrent) length() int64 { @@ -496,7 +502,7 @@ func (t *Torrent) setInfo(info *metainfo.Info) error { } if t.storageOpener != nil { var err error - t.storage, err = t.storageOpener.OpenTorrent(info, t.infoHash) + t.storage, err = t.storageOpener.OpenTorrent(info, *t.canonicalShortInfohash()) if err != nil { return fmt.Errorf("error opening torrent storage: %s", err) } @@ -515,7 +521,7 @@ func (t *Torrent) setInfo(info *metainfo.Info) error { func (t *Torrent) pieceRequestOrderKey(i int) request_strategy.PieceRequestOrderKey { return request_strategy.PieceRequestOrderKey{ - InfoHash: t.infoHash, + InfoHash: *t.canonicalShortInfohash(), Index: i, } } @@ -550,18 +556,23 @@ func (t *Torrent) onSetInfo() { // Called when metadata for a torrent becomes available. func (t *Torrent) setInfoBytesLocked(b []byte) error { - v2Hash := infohash_v2.HashBytes(b) - v1Hash := infohash.HashBytes(b) - v2Short := v2Hash.ToShort() - if v2Short != t.infoHash && v1Hash != t.infoHash { - return errors.New("info bytes have wrong hash") + if t.infoHash.Ok && infohash.HashBytes(b) != t.infoHash.Value { + return errors.New("info bytes have wrong v1 hash") + } + var v2Hash g.Option[infohash_v2.T] + if t.infoHashV2.Ok { + v2Hash.Set(infohash_v2.HashBytes(b)) + if v2Hash.Value != t.infoHashV2.Value { + return errors.New("info bytes have wrong v2 hash") + } } var info metainfo.Info if err := bencode.Unmarshal(b, &info); err != nil { return fmt.Errorf("error unmarshalling info bytes: %s", err) } - if info.HasV2() { - t.infoHashV2.Set(v2Hash) + if !t.infoHashV2.Ok && info.HasV2() { + v2Hash.Set(infohash_v2.HashBytes(b)) + t.infoHashV2.Set(v2Hash.Unwrap()) } t.metadataBytes = b t.metadataCompletedChunks = nil @@ -622,7 +633,7 @@ func (t *Torrent) name() string { if t.displayName != "" { return t.displayName } - return "infohash:" + t.infoHash.HexString() + return "infohash:" + t.canonicalShortInfohash().HexString() } func (t *Torrent) pieceState(index pieceIndex) (ret PieceState) { @@ -738,7 +749,12 @@ func (psr PieceStateRun) String() (ret string) { } func (t *Torrent) writeStatus(w io.Writer) { - fmt.Fprintf(w, "Infohash: %s\n", t.infoHash.HexString()) + if t.infoHash.Ok { + fmt.Fprintf(w, "Infohash: %s\n", t.infoHash.Value.HexString()) + } + if t.infoHashV2.Ok { + fmt.Fprintf(w, "Infohash v2: %s\n", t.infoHashV2.Value.HexString()) + } fmt.Fprintf(w, "Metadata length: %d\n", t.metadataSize()) if !t.haveInfo() { fmt.Fprintf(w, "Metadata have: ") @@ -1789,14 +1805,14 @@ func (t *Torrent) runHandshookConnLoggingErr(pc *PeerConn) { t.logRunHandshookConn(pc, false, log.Debug) } -func (t *Torrent) startWebsocketAnnouncer(u url.URL) torrentTrackerAnnouncer { - wtc, release := t.cl.websocketTrackers.Get(u.String(), t.infoHash) - // This needs to run before the Torrent is dropped from the Client, to prevent a new webtorrent.TrackerClient for - // the same info hash before the old one is cleaned up. +func (t *Torrent) startWebsocketAnnouncer(u url.URL, shortInfohash [20]byte) torrentTrackerAnnouncer { + wtc, release := t.cl.websocketTrackers.Get(u.String(), shortInfohash) + // This needs to run before the Torrent is dropped from the Client, to prevent a new + // webtorrent.TrackerClient for the same info hash before the old one is cleaned up. t.onClose = append(t.onClose, release) wst := websocketTrackerStatus{u, wtc} go func() { - err := wtc.Announce(tracker.Started, t.infoHash) + err := wtc.Announce(tracker.Started, shortInfohash) if err != nil { t.logger.WithDefaultLevel(log.Warning).Printf( "error in initial announce to %q: %v", @@ -1826,7 +1842,20 @@ func (t *Torrent) startScrapingTracker(_url string) { t.startScrapingTracker(u.String()) return } - if _, ok := t.trackerAnnouncers[_url]; ok { + if t.infoHash.Ok { + t.startScrapingTrackerWithInfohash(u, _url, t.infoHash.Value) + } + if t.infoHashV2.Ok { + t.startScrapingTrackerWithInfohash(u, _url, *t.infoHashV2.Value.ToShort()) + } +} + +func (t *Torrent) startScrapingTrackerWithInfohash(u *url.URL, urlStr string, shortInfohash [20]byte) { + announcerKey := torrentTrackerAnnouncerKey{ + shortInfohash: shortInfohash, + url: urlStr, + } + if _, ok := t.trackerAnnouncers[announcerKey]; ok { return } sl := func() torrentTrackerAnnouncer { @@ -1835,7 +1864,7 @@ func (t *Torrent) startScrapingTracker(_url string) { if t.cl.config.DisableWebtorrent { return nil } - return t.startWebsocketAnnouncer(*u) + return t.startWebsocketAnnouncer(*u, shortInfohash) case "udp4": if t.cl.config.DisableIPv4Peers || t.cl.config.DisableIPv4 { return nil @@ -1846,6 +1875,7 @@ func (t *Torrent) startScrapingTracker(_url string) { } } newAnnouncer := &trackerScraper{ + shortInfohash: shortInfohash, u: *u, t: t, lookupTrackerIp: t.cl.config.LookupTrackerIp, @@ -1856,10 +1886,10 @@ func (t *Torrent) startScrapingTracker(_url string) { if sl == nil { return } - if t.trackerAnnouncers == nil { - t.trackerAnnouncers = make(map[string]torrentTrackerAnnouncer) + g.MakeMapIfNil(&t.trackerAnnouncers) + if g.MapInsert(t.trackerAnnouncers, announcerKey, sl).Ok { + panic("tracker announcer already exists") } - t.trackerAnnouncers[_url] = sl } // Adds and starts tracker scrapers for tracker URLs that aren't already @@ -1878,21 +1908,26 @@ func (t *Torrent) startMissingTrackerScrapers() { // Returns an AnnounceRequest with fields filled out to defaults and current // values. -func (t *Torrent) announceRequest(event tracker.AnnounceEvent) tracker.AnnounceRequest { +func (t *Torrent) announceRequest( + event tracker.AnnounceEvent, + shortInfohash [20]byte, +) tracker.AnnounceRequest { // Note that IPAddress is not set. It's set for UDP inside the tracker code, since it's // dependent on the network in use. return tracker.AnnounceRequest{ Event: event, NumWant: func() int32 { if t.wantPeers() && len(t.cl.dialers) > 0 { - return 200 // Win has UDP packet limit. See: https://github.com/anacrolix/torrent/issues/764 + // Windozer has UDP packet limit. See: + // https://github.com/anacrolix/torrent/issues/764 + return 200 } else { return 0 } }(), Port: uint16(t.cl.incomingPeerPort()), PeerId: t.cl.peerID, - InfoHash: t.infoHash, + InfoHash: shortInfohash, Key: t.cl.announceKey(), // The following are vaguely described in BEP 3. @@ -1931,22 +1966,62 @@ func (t *Torrent) consumeDhtAnnouncePeers(pvs <-chan dht.PeersValues) { } // Announce using the provided DHT server. Peers are consumed automatically. done is closed when the -// announce ends. stop will force the announce to end. +// announce ends. stop will force the announce to end. This interface is really old-school, and +// calls a private one that is much more modern. Both v1 and v2 info hashes are announced if they +// exist. func (t *Torrent) AnnounceToDht(s DhtServer) (done <-chan struct{}, stop func(), err error) { - ps, err := s.Announce(t.infoHash, t.cl.incomingPeerPort(), true) - if err != nil { - return + var ihs [][20]byte + t.cl.lock() + t.eachShortInfohash(func(short [20]byte) { + ihs = append(ihs, short) + }) + t.cl.unlock() + ctx, stop := context.WithCancel(context.Background()) + eg, ctx := errgroup.WithContext(ctx) + for _, ih := range ihs { + var ann DhtAnnounce + ann, err = s.Announce(ih, t.cl.incomingPeerPort(), true) + if err != nil { + stop() + return + } + eg.Go(func() error { + return t.dhtAnnounceConsumer(ctx, ann) + }) } _done := make(chan struct{}) done = _done - stop = ps.Close go func() { - t.consumeDhtAnnouncePeers(ps.Peers()) - close(_done) + defer stop() + defer close(_done) + // Won't this race? + err = eg.Wait() }() return } +// Announce using the provided DHT server. Peers are consumed automatically. done is closed when the +// announce ends. stop will force the announce to end. +func (t *Torrent) dhtAnnounceConsumer( + ctx context.Context, + ps DhtAnnounce, +) ( + err error, +) { + defer ps.Close() + done := make(chan struct{}) + go func() { + defer close(done) + t.consumeDhtAnnouncePeers(ps.Peers()) + }() + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-done: + return nil + } +} + func (t *Torrent) timeboxedAnnounceToDht(s DhtServer) error { _, stop, err := t.AnnounceToDht(s) if err != nil { @@ -3015,3 +3090,19 @@ func (t *Torrent) getDialTimeoutUnlocked() time.Duration { defer cl.rUnlock() return t.dialTimeout() } + +func (t *Torrent) canonicalShortInfohash() *infohash.T { + if t.infoHash.Ok { + return &t.infoHash.Value + } + return t.infoHashV2.UnwrapPtr().ToShort() +} + +func (t *Torrent) eachShortInfohash(each func(short [20]byte)) { + if t.infoHash.Ok { + each(t.infoHash.Value) + } + if t.infoHashV2.Ok { + each(*t.infoHashV2.Value.ToShort()) + } +} diff --git a/torrent_test.go b/torrent_test.go index 808947e974..15c2a4fc34 100644 --- a/torrent_test.go +++ b/torrent_test.go @@ -71,8 +71,10 @@ func TestAppendToCopySlice(t *testing.T) { func TestTorrentString(t *testing.T) { tor := &Torrent{} + tor.infoHash.Ok = true + tor.infoHash.Value[0] = 1 s := tor.InfoHash().HexString() - if s != "0000000000000000000000000000000000000000" { + if s != "0100000000000000000000000000000000000000" { t.FailNow() } } @@ -87,7 +89,7 @@ func BenchmarkUpdatePiecePriorities(b *testing.B) { ) cl := &Client{config: TestingConfig(b)} cl.initLogger() - t := cl.newTorrent(metainfo.Hash{}, nil) + t := cl.newTorrentForTesting() require.NoError(b, t.setInfo(&metainfo.Info{ Pieces: make([]byte, metainfo.HashSize*numPieces), PieceLength: pieceLength, diff --git a/tracker_scraper.go b/tracker_scraper.go index 863838ace4..0668c9127e 100644 --- a/tracker_scraper.go +++ b/tracker_scraper.go @@ -18,6 +18,7 @@ import ( // Announces a torrent to a tracker at regular intervals, when peers are // required. type trackerScraper struct { + shortInfohash [20]byte u url.URL t *Torrent lastAnnounce trackerAnnounceResult @@ -117,7 +118,10 @@ func (me *trackerScraper) trackerUrl(ip net.IP) string { // Return how long to wait before trying again. For most errors, we return 5 // minutes, a relatively quick turn around for DNS changes. -func (me *trackerScraper) announce(ctx context.Context, event tracker.AnnounceEvent) (ret trackerAnnounceResult) { +func (me *trackerScraper) announce( + ctx context.Context, + event tracker.AnnounceEvent, +) (ret trackerAnnounceResult) { defer func() { ret.Completed = time.Now() }() @@ -146,7 +150,7 @@ func (me *trackerScraper) announce(ctx context.Context, event tracker.AnnounceEv return } me.t.cl.rLock() - req := me.t.announceRequest(event) + req := me.t.announceRequest(event, me.shortInfohash) me.t.cl.rUnlock() // The default timeout works well as backpressure on concurrent access to the tracker. Since // we're passing our own Context now, we will include that timeout ourselves to maintain similar diff --git a/types/infohash-v2/infohash-v2.go b/types/infohash-v2/infohash-v2.go index 02ddd1d8f2..c9d00732c0 100644 --- a/types/infohash-v2/infohash-v2.go +++ b/types/infohash-v2/infohash-v2.go @@ -5,6 +5,7 @@ import ( "encoding" "encoding/hex" "fmt" + "unsafe" "github.com/multiformats/go-multihash" @@ -56,9 +57,8 @@ func (t *T) FromHexString(s string) (err error) { } // Truncates the hash to 20 bytes for use in auxiliary interfaces, like DHT and trackers. -func (t *T) ToShort() (short infohash.T) { - copy(short[:], t[:]) - return +func (t *T) ToShort() (short *infohash.T) { + return (*infohash.T)(unsafe.Pointer(t)) } var ( diff --git a/types/infohash/infohash.go b/types/infohash/infohash.go index c90f7451e4..30f68a241e 100644 --- a/types/infohash/infohash.go +++ b/types/infohash/infohash.go @@ -52,6 +52,10 @@ func (t *T) FromHexString(s string) (err error) { return } +func (t *T) IsZero() bool { + return *t == T{} +} + var ( _ encoding.TextUnmarshaler = (*T)(nil) _ encoding.TextMarshaler = T{} From 106c9163b85cde871086aba6bb41767012c9e9a0 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 1 Mar 2024 00:11:13 +1100 Subject: [PATCH 09/15] Send hash requests for missing v2 hashes --- merkle/merkle.go | 4 +++ misc.go | 10 ------ peer_protocol/msg.go | 31 +++++++++++++++--- peerconn.go | 77 ++++++++++++++++++++++++++++++++++++++++++++ piecestate.go | 3 ++ torrent.go | 12 +++++++ 6 files changed, 122 insertions(+), 15 deletions(-) diff --git a/merkle/merkle.go b/merkle/merkle.go index ab54af6a2b..171555510d 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -47,3 +47,7 @@ func CompactLayerToSliceHashes(compactLayer string) (hashes [][sha256.Size]byte, func RoundUpToPowerOfTwo(n uint) (ret uint) { return 1 << bits.Len(n-1) } + +func Log2RoundingUp(n uint) (ret uint) { + return uint(bits.Len(n - 1)) +} diff --git a/misc.go b/misc.go index 8f82c2a0f2..42c516f0ea 100644 --- a/misc.go +++ b/misc.go @@ -157,16 +157,6 @@ func maxInt(as ...int) int { return ret } -func min(as ...int64) int64 { - ret := as[0] - for _, a := range as[1:] { - if a < ret { - ret = a - } - } - return ret -} - func minInt(as ...int) int { ret := as[0] for _, a := range as[1:] { diff --git a/peer_protocol/msg.go b/peer_protocol/msg.go index f1b1f10e83..b08bb5380e 100644 --- a/peer_protocol/msg.go +++ b/peer_protocol/msg.go @@ -9,16 +9,20 @@ import ( ) // This is a lazy union representing all the possible fields for messages. Go doesn't have ADTs, and -// I didn't choose to use type-assertions. +// I didn't choose to use type-assertions. Fields are ordered to minimize struct size and padding. type Message struct { - Keepalive bool - Type MessageType - Index, Begin, Length Integer + PiecesRoot [32]byte Piece []byte Bitfield []bool - ExtendedID ExtensionNumber ExtendedPayload []byte + Hashes [][32]byte + Index, Begin, Length Integer + BaseLayer Integer + ProofLayers Integer Port uint16 + Type MessageType + ExtendedID ExtensionNumber + Keepalive bool } var _ interface { @@ -58,7 +62,21 @@ func (msg Message) MustMarshalBinary() []byte { } func (msg Message) MarshalBinary() (data []byte, err error) { + // It might look like you could have a pool of buffers and preallocate the message length + // prefix, but because we have to return []byte, it becomes non-trivial to make this fast. You + // will need a benchmark. var buf bytes.Buffer + mustWrite := func(data any) { + err := binary.Write(&buf, binary.BigEndian, data) + if err != nil { + panic(err) + } + } + writeConsecutive := func(data ...any) { + for _, d := range data { + mustWrite(d) + } + } if !msg.Keepalive { err = buf.WriteByte(byte(msg.Type)) if err != nil { @@ -99,6 +117,9 @@ func (msg Message) MarshalBinary() (data []byte, err error) { _, err = buf.Write(msg.ExtendedPayload) case Port: err = binary.Write(&buf, binary.BigEndian, msg.Port) + case HashRequest: + buf.Write(msg.PiecesRoot[:]) + writeConsecutive(msg.BaseLayer, msg.Index, msg.Length, msg.ProofLayers) default: err = fmt.Errorf("unknown message type: %v", msg.Type) } diff --git a/peerconn.go b/peerconn.go index 59a31ad4a7..ea2ee9662d 100644 --- a/peerconn.go +++ b/peerconn.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "github.com/anacrolix/torrent/merkle" "io" "math/rand" "net" @@ -79,6 +80,8 @@ type PeerConn struct { peerRequestDataAllocLimiter alloclim.Limiter outstandingHolepunchingRendezvous map[netip.AddrPort]struct{} + + sentHashRequests map[hashRequest]struct{} } func (cn *PeerConn) pexStatus() string { @@ -336,6 +339,7 @@ func (cn *PeerConn) fillWriteBuffer() { // knowledge of write buffers. return } + cn.requestMissingHashes() cn.maybeUpdateActualRequestState() if cn.pex.IsEnabled() { if flow := cn.pex.Share(cn.write); !flow { @@ -1214,3 +1218,76 @@ func (pc *PeerConn) WriteExtendedMessage(extName pp.ExtensionName, payload []byt }) return nil } + +func (pc *PeerConn) requestMissingHashes() { + if !pc.t.haveInfo() { + return + } + info := pc.t.info + if !info.HasV2() { + return + } + baseLayer := pp.Integer(merkle.Log2RoundingUp(merkle.RoundUpToPowerOfTwo( + uint((pc.t.usualPieceSize() + merkle.BlockSize - 1) / merkle.BlockSize)), + )) + for _, file := range info.UpvertedFiles() { + piecesRoot := file.PiecesRoot.Unwrap() + fileNumPieces := int((file.Length + info.PieceLength - 1) / info.PieceLength) + proofLayers := pp.Integer(0) + // We would be requesting the leaves, the file must be short enough that we can just do with + // the pieces root as the piece hash. + if fileNumPieces <= 1 { + continue + } + for index := 0; index < fileNumPieces; index += 512 { + // Minimizing to the number of pieces in a file conflicts with the BEP. + length := merkle.RoundUpToPowerOfTwo(uint(min(512, fileNumPieces-index))) + if length < 2 { + // This should have been filtered out by baseLayer and pieces root as piece hash + // checks. + panic(length) + } + msg := pp.Message{ + Type: pp.HashRequest, + PiecesRoot: piecesRoot, + BaseLayer: baseLayer, + Index: pp.Integer(index), + Length: pp.Integer(length), + ProofLayers: proofLayers, + } + hr := hashRequestFromMessage(msg) + if generics.MapContains(pc.sentHashRequests, hr) { + continue + } + pc.write(msg) + generics.MakeMapIfNil(&pc.sentHashRequests) + pc.sentHashRequests[hr] = struct{}{} + } + } +} + +type hashRequest struct { + piecesRoot [32]byte + baseLayer, index, length, proofLayers pp.Integer +} + +func (hr hashRequest) toMessage() pp.Message { + return pp.Message{ + Type: pp.HashRequest, + PiecesRoot: hr.piecesRoot, + BaseLayer: hr.baseLayer, + Index: hr.index, + Length: hr.length, + ProofLayers: hr.proofLayers, + } +} + +func hashRequestFromMessage(m pp.Message) hashRequest { + return hashRequest{ + piecesRoot: m.PiecesRoot, + baseLayer: m.BaseLayer, + index: m.Index, + length: m.Length, + proofLayers: m.ProofLayers, + } +} diff --git a/piecestate.go b/piecestate.go index 089adca440..9e67907bd9 100644 --- a/piecestate.go +++ b/piecestate.go @@ -18,6 +18,9 @@ type PieceState struct { // Some of the piece has been obtained. Partial bool + + // The v2 hash for the piece layer is missing. + MissingPieceLayerHash bool } // Represents a series of consecutive pieces with the same state. diff --git a/torrent.go b/torrent.go index 3c925ae577..8a403e16ab 100644 --- a/torrent.go +++ b/torrent.go @@ -416,6 +416,12 @@ func (t *Torrent) makePieces() { if numFiles != 1 { panic(fmt.Sprintf("%v:%v", beginFile, endFile)) } + if t.info.HasV2() { + file := piece.mustGetOnlyFile() + if file.numPieces() == 1 { + piece.hashV2.Set(file.piecesRoot.Unwrap()) + } + } } } } @@ -647,6 +653,9 @@ func (t *Torrent) pieceState(index pieceIndex) (ret PieceState) { if !ret.Complete && t.piecePartiallyDownloaded(index) { ret.Partial = true } + if t.info.HasV2() && !p.hashV2.Ok { + ret.MissingPieceLayerHash = true + } return } @@ -745,6 +754,9 @@ func (psr PieceStateRun) String() (ret string) { if !psr.Ok { ret += "?" } + if psr.MissingPieceLayerHash { + ret += "h" + } return } From c00e6f51ba4c03218f3d7ad83fce62c873e97fd5 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 1 Mar 2024 14:22:41 +1100 Subject: [PATCH 10/15] Implement decoding hash request, reject and hashes --- peer_protocol/decoder.go | 180 ++++++++++++++++++++-------- peer_protocol/fuzz_test.go | 4 +- peer_protocol/messagetype_string.go | 33 ++++- peer_protocol/protocol.go | 7 +- v2hashes.go | 1 + 5 files changed, 165 insertions(+), 60 deletions(-) create mode 100644 v2hashes.go diff --git a/peer_protocol/decoder.go b/peer_protocol/decoder.go index 9dfe125b1a..49eda43699 100644 --- a/peer_protocol/decoder.go +++ b/peer_protocol/decoder.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/binary" "fmt" + g "github.com/anacrolix/generics" "io" "sync" @@ -19,41 +20,109 @@ type Decoder struct { MaxLength Integer // TODO: Should this include the length header or not? } -// io.EOF is returned if the source terminates cleanly on a message boundary. -func (d *Decoder) Decode(msg *Message) (err error) { - var length Integer - err = length.Read(d.R) - if err != nil { - return fmt.Errorf("reading message length: %w", err) +// This limits reads to the length of a message, returning io.EOF when the end of the message bytes +// are reached. If you aren't expecting io.EOF, you should probably wrap it with expectReader. +type decodeReader struct { + lr io.LimitedReader + br *bufio.Reader +} + +func (dr *decodeReader) Init(r *bufio.Reader, length int64) { + dr.lr.R = r + dr.lr.N = length + dr.br = r +} + +func (dr *decodeReader) ReadByte() (c byte, err error) { + if dr.lr.N <= 0 { + err = io.EOF + return } - if length > d.MaxLength { - return errors.New("message too long") + c, err = dr.br.ReadByte() + if err == nil { + dr.lr.N-- } - if length == 0 { - msg.Keepalive = true - return + return +} + +func (dr *decodeReader) Read(p []byte) (n int, err error) { + n, err = dr.lr.Read(p) + if dr.lr.N != 0 && err == io.EOF { + err = io.ErrUnexpectedEOF + } + return +} + +func (dr *decodeReader) UnreadLength() int64 { + return dr.lr.N +} + +// This expects reads to have enough bytes. io.EOF is mapped to io.ErrUnexpectedEOF. It's probably +// not a good idea to pass this to functions that expect to read until the end of something, because +// they will probably expect io.EOF. +type expectReader struct { + dr *decodeReader +} + +func (er expectReader) ReadByte() (c byte, err error) { + c, err = er.dr.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF } - r := d.R - readByte := func() (byte, error) { - length-- - return d.R.ReadByte() + return +} + +func (er expectReader) Read(p []byte) (n int, err error) { + n, err = er.dr.Read(p) + if err == io.EOF { + err = io.ErrUnexpectedEOF } - // From this point onwards, EOF is unexpected - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF + return +} + +func (er expectReader) UnreadLength() int64 { + return er.dr.UnreadLength() +} + +// io.EOF is returned if the source terminates cleanly on a message boundary. +func (d *Decoder) Decode(msg *Message) (err error) { + var dr decodeReader + { + var length Integer + err = length.Read(d.R) + if err != nil { + return fmt.Errorf("reading message length: %w", err) } - }() - c, err := readByte() + if length > d.MaxLength { + return errors.New("message too long") + } + if length == 0 { + msg.Keepalive = true + return + } + dr.Init(d.R, int64(length)) + } + r := expectReader{&dr} + c, err := r.ReadByte() if err != nil { return } msg.Type = MessageType(c) - // Can return directly in cases when err is not nil, or length is known to be zero. + err = readMessageAfterType(msg, &r, d.Pool) + if err != nil { + err = fmt.Errorf("reading fields for message type %v: %w", msg.Type, err) + return + } + if r.UnreadLength() != 0 { + err = fmt.Errorf("%v unused bytes in message type %v", r.UnreadLength(), msg.Type) + } + return +} + +func readMessageAfterType(msg *Message, r *expectReader, piecePool *sync.Pool) (err error) { switch msg.Type { case Choke, Unchoke, Interested, NotInterested, HaveAll, HaveNone: case Have, AllowedFast, Suggest: - length -= 4 err = msg.Index.Read(r) case Request, Cancel, Reject: for _, data := range []*Integer{&msg.Index, &msg.Begin, &msg.Length} { @@ -62,67 +131,74 @@ func (d *Decoder) Decode(msg *Message) (err error) { break } } - length -= 12 case Bitfield: - b := make([]byte, length) + b := make([]byte, r.UnreadLength()) _, err = io.ReadFull(r, b) - length = 0 msg.Bitfield = unmarshalBitfield(b) - return case Piece: for _, pi := range []*Integer{&msg.Index, &msg.Begin} { - err := pi.Read(r) + err = pi.Read(r) if err != nil { - return err + return } } - length -= 8 - dataLen := int64(length) - if d.Pool == nil { + dataLen := r.UnreadLength() + if piecePool == nil { msg.Piece = make([]byte, dataLen) } else { - msg.Piece = *d.Pool.Get().(*[]byte) + msg.Piece = *piecePool.Get().(*[]byte) if int64(cap(msg.Piece)) < dataLen { return errors.New("piece data longer than expected") } msg.Piece = msg.Piece[:dataLen] } _, err = io.ReadFull(r, msg.Piece) - length = 0 - return case Extended: var b byte - b, err = readByte() + b, err = r.ReadByte() if err != nil { break } msg.ExtendedID = ExtensionNumber(b) - msg.ExtendedPayload = make([]byte, length) + msg.ExtendedPayload = make([]byte, r.UnreadLength()) _, err = io.ReadFull(r, msg.ExtendedPayload) - length = 0 - return case Port: err = binary.Read(r, binary.BigEndian, &msg.Port) - length -= 2 + case HashRequest, HashReject: + err = readHashRequest(r, msg) + case Hashes: + err = readHashRequest(r, msg) + numHashes := (r.UnreadLength() + 31) / 32 + g.MakeSliceWithCap(&msg.Hashes, numHashes) + for range numHashes { + var oneHash [32]byte + _, err = io.ReadFull(r, oneHash[:]) + if err != nil { + err = fmt.Errorf("error while reading hashes: %w", err) + return + } + msg.Hashes = append(msg.Hashes, oneHash) + } default: - err = fmt.Errorf("unknown message type %#v", c) - } - if err == nil && length != 0 { - err = fmt.Errorf("%v unused bytes in message type %v", length, msg.Type) + err = errors.New("unhandled message type") } return } -func readByte(r io.Reader) (b byte, err error) { - var arr [1]byte - n, err := r.Read(arr[:]) - b = arr[0] - if n == 1 { - err = nil +func readHashRequest(r io.Reader, msg *Message) (err error) { + _, err = io.ReadFull(r, msg.PiecesRoot[:]) + if err != nil { return } - if err == nil { - panic(err) + return readSeq(r, &msg.BaseLayer, &msg.Index, &msg.Length, &msg.ProofLayers) +} + +func readSeq(r io.Reader, data ...any) (err error) { + for _, d := range data { + err = binary.Read(r, binary.BigEndian, d) + if err != nil { + return + } } return } diff --git a/peer_protocol/fuzz_test.go b/peer_protocol/fuzz_test.go index 5241504853..8ffdfd7b47 100644 --- a/peer_protocol/fuzz_test.go +++ b/peer_protocol/fuzz_test.go @@ -6,7 +6,6 @@ package peer_protocol import ( "bufio" "bytes" - "errors" "io" "testing" @@ -30,7 +29,7 @@ func FuzzDecoder(f *testing.F) { var m Message err := d.Decode(&m) t.Log(err) - if errors.Is(err, io.EOF) { + if err == io.EOF { break } if err == nil { @@ -41,6 +40,7 @@ func FuzzDecoder(f *testing.F) { t.Skip(err) } } + t.Log(ms) var buf bytes.Buffer for _, m := range ms { buf.Write(m.MustMarshalBinary()) diff --git a/peer_protocol/messagetype_string.go b/peer_protocol/messagetype_string.go index 7be19f4275..e1ad6a88a8 100644 --- a/peer_protocol/messagetype_string.go +++ b/peer_protocol/messagetype_string.go @@ -4,15 +4,41 @@ package peer_protocol import "strconv" +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Choke-0] + _ = x[Unchoke-1] + _ = x[Interested-2] + _ = x[NotInterested-3] + _ = x[Have-4] + _ = x[Bitfield-5] + _ = x[Request-6] + _ = x[Piece-7] + _ = x[Cancel-8] + _ = x[Port-9] + _ = x[Suggest-13] + _ = x[HaveAll-14] + _ = x[HaveNone-15] + _ = x[Reject-16] + _ = x[AllowedFast-17] + _ = x[Extended-20] + _ = x[HashRequest-21] + _ = x[Hashes-22] + _ = x[HashReject-23] +} + const ( _MessageType_name_0 = "ChokeUnchokeInterestedNotInterestedHaveBitfieldRequestPieceCancelPort" _MessageType_name_1 = "SuggestHaveAllHaveNoneRejectAllowedFast" - _MessageType_name_2 = "Extended" + _MessageType_name_2 = "ExtendedHashRequestHashesHashReject" ) var ( _MessageType_index_0 = [...]uint8{0, 5, 12, 22, 35, 39, 47, 54, 59, 65, 69} _MessageType_index_1 = [...]uint8{0, 7, 14, 22, 28, 39} + _MessageType_index_2 = [...]uint8{0, 8, 19, 25, 35} ) func (i MessageType) String() string { @@ -22,8 +48,9 @@ func (i MessageType) String() string { case 13 <= i && i <= 17: i -= 13 return _MessageType_name_1[_MessageType_index_1[i]:_MessageType_index_1[i+1]] - case i == 20: - return _MessageType_name_2 + case 20 <= i && i <= 23: + i -= 20 + return _MessageType_name_2[_MessageType_index_2[i]:_MessageType_index_2[i+1]] default: return "MessageType(" + strconv.FormatInt(int64(i), 10) + ")" } diff --git a/peer_protocol/protocol.go b/peer_protocol/protocol.go index 2f92ab324d..b1790d1c10 100644 --- a/peer_protocol/protocol.go +++ b/peer_protocol/protocol.go @@ -6,6 +6,7 @@ const ( type MessageType byte +// golang.org/x/tools/cmd/stringer //go:generate stringer -type=MessageType func (mt MessageType) FastExtension() bool { @@ -43,9 +44,9 @@ const ( Extended MessageType = 20 // BEP 52 - HashRequest = 21 - Hashes = 22 - HashReject = 23 + HashRequest MessageType = 21 + Hashes MessageType = 22 + HashReject MessageType = 23 ) const ( diff --git a/v2hashes.go b/v2hashes.go new file mode 100644 index 0000000000..10cbafc73d --- /dev/null +++ b/v2hashes.go @@ -0,0 +1 @@ +package torrent From c4e0e154400ab76ca4bde9d0b88d6274f1e9168f Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 1 Mar 2024 14:42:22 +1100 Subject: [PATCH 11/15] Implement reading piece hashes from peers --- go.mod | 2 +- merkle/hash.go | 4 +-- merkle/merkle.go | 7 +++++ metainfo/bep52.go | 7 +---- peerconn.go | 77 +++++++++++++++++++++++++++++++++++++++++++++-- torrent.go | 12 ++++++-- v2hashes.go | 4 +++ 7 files changed, 98 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 637f205a2d..1af03a6edd 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( go.opentelemetry.io/otel/sdk v1.8.0 go.opentelemetry.io/otel/trace v1.8.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df + golang.org/x/sync v0.3.0 golang.org/x/sys v0.15.0 golang.org/x/time v0.0.0-20220609170525-579cf78fd858 ) @@ -111,7 +112,6 @@ require ( go.opentelemetry.io/proto/otlp v0.18.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect diff --git a/merkle/hash.go b/merkle/hash.go index 18ecee78e0..1a8f67a922 100644 --- a/merkle/hash.go +++ b/merkle/hash.go @@ -50,9 +50,7 @@ func (h *Hash) Sum(b []byte) []byte { if h.written != 0 { blocks = append(blocks, h.nextBlockSum()) } - n := int(RoundUpToPowerOfTwo(uint(len(blocks)))) - blocks = append(blocks, make([][32]byte, n-len(blocks))...) - sum := Root(blocks) + sum := RootWithPadHash(blocks, [32]byte{}) return append(b, sum[:]...) } diff --git a/merkle/merkle.go b/merkle/merkle.go index 171555510d..582c08074b 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -32,6 +32,13 @@ func Root(hashes [][sha256.Size]byte) [sha256.Size]byte { return Root(next) } +func RootWithPadHash(hashes [][sha256.Size]byte, padHash [sha256.Size]byte) [sha256.Size]byte { + for uint(len(hashes)) < RoundUpToPowerOfTwo(uint(len(hashes))) { + hashes = append(hashes, padHash) + } + return Root(hashes) +} + func CompactLayerToSliceHashes(compactLayer string) (hashes [][sha256.Size]byte, err error) { g.MakeSliceWithLength(&hashes, len(compactLayer)/sha256.Size) for i := range hashes { diff --git a/metainfo/bep52.go b/metainfo/bep52.go index 0291d65349..dd3f741432 100644 --- a/metainfo/bep52.go +++ b/metainfo/bep52.go @@ -36,12 +36,7 @@ func ValidatePieceLayers( } var layerHashes [][32]byte layerHashes, err = merkle.CompactLayerToSliceHashes(filePieceLayers) - padHash := HashForPiecePad(pieceLength) - for uint(len(layerHashes)) < merkle.RoundUpToPowerOfTwo(uint(len(layerHashes))) { - layerHashes = append(layerHashes, padHash) - } - var root [32]byte - root = merkle.Root(layerHashes) + root := merkle.RootWithPadHash(layerHashes, HashForPiecePad(pieceLength)) if root != piecesRoot.Value { err = fmt.Errorf("file %q: expected hash %x got %x", path, piecesRoot.Value, root) return diff --git a/peerconn.go b/peerconn.go index ea2ee9662d..c55a7d7321 100644 --- a/peerconn.go +++ b/peerconn.go @@ -81,7 +81,13 @@ type PeerConn struct { outstandingHolepunchingRendezvous map[netip.AddrPort]struct{} + // Hash requests sent to the peer. If there's an issue we probably don't want to reissue these, + // because I haven't implemented it smart enough yet. sentHashRequests map[hashRequest]struct{} + // Hash pieces received from the peer, mapped from pieces root to piece layer hashes. This way + // we can verify all the pieces for a file when they're all arrived before submitting them to + // the torrent. + receivedHashPieces map[[32]byte][][32]byte } func (cn *PeerConn) pexStatus() string { @@ -738,7 +744,11 @@ func (c *PeerConn) mainReadLoop() (err error) { cl.unlock() defer cl.lock() err = decoder.Decode(&msg) + if err != nil { + err = fmt.Errorf("decoding message: %w", err) + } }() + // Do this before checking closed. if cb := c.callbacks.ReadMessage; cb != nil && err == nil { cb(c, &msg) } @@ -746,6 +756,7 @@ func (c *PeerConn) mainReadLoop() (err error) { return nil } if err != nil { + err = log.WithLevel(log.Info, err) return err } c.lastMessageReceived = time.Now() @@ -869,7 +880,9 @@ func (c *PeerConn) mainReadLoop() (err error) { c.updateRequests("PeerConn.mainReadLoop allowed fast") case pp.Extended: err = c.onReadExtendedMsg(msg.ExtendedID, msg.ExtendedPayload) - case pp.HashRequest, pp.Hashes, pp.HashReject: + case pp.Hashes: + err = c.onReadHashes(&msg) + case pp.HashRequest, pp.HashReject: err = log.WithLevel(log.Warning, fmt.Errorf("received unimplemented BitTorrent v2 message: %v", msg.Type)) default: err = fmt.Errorf("received unknown message type: %#v", msg.Type) @@ -1220,6 +1233,9 @@ func (pc *PeerConn) WriteExtendedMessage(extName pp.ExtensionName, payload []byt } func (pc *PeerConn) requestMissingHashes() { + if pc.peerChoking { + return + } if !pc.t.haveInfo() { return } @@ -1230,15 +1246,32 @@ func (pc *PeerConn) requestMissingHashes() { baseLayer := pp.Integer(merkle.Log2RoundingUp(merkle.RoundUpToPowerOfTwo( uint((pc.t.usualPieceSize() + merkle.BlockSize - 1) / merkle.BlockSize)), )) + nextFileBeginPiece := 0 +file: for _, file := range info.UpvertedFiles() { - piecesRoot := file.PiecesRoot.Unwrap() fileNumPieces := int((file.Length + info.PieceLength - 1) / info.PieceLength) - proofLayers := pp.Integer(0) + curFileBeginPiece := nextFileBeginPiece + nextFileBeginPiece += fileNumPieces + haveAllHashes := true + for i := range fileNumPieces { + torrentPieceIndex := curFileBeginPiece + i + if !pc.peerHasPiece(torrentPieceIndex) { + continue file + } + if !pc.t.piece(torrentPieceIndex).hashV2.Ok { + haveAllHashes = false + } + } + if haveAllHashes { + continue + } // We would be requesting the leaves, the file must be short enough that we can just do with // the pieces root as the piece hash. if fileNumPieces <= 1 { continue } + piecesRoot := file.PiecesRoot.Unwrap() + proofLayers := pp.Integer(0) for index := 0; index < fileNumPieces; index += 512 { // Minimizing to the number of pieces in a file conflicts with the BEP. length := merkle.RoundUpToPowerOfTwo(uint(min(512, fileNumPieces-index))) @@ -1247,6 +1280,9 @@ func (pc *PeerConn) requestMissingHashes() { // checks. panic(length) } + if length%2 != 0 { + pc.logger.Levelf(log.Warning, "requesting odd hashes length %d", length) + } msg := pp.Message{ Type: pp.HashRequest, PiecesRoot: piecesRoot, @@ -1266,6 +1302,41 @@ func (pc *PeerConn) requestMissingHashes() { } } +func (pc *PeerConn) onReadHashes(msg *pp.Message) (err error) { + file := pc.t.getFileByPiecesRoot(msg.PiecesRoot) + filePieceHashes := pc.receivedHashPieces[msg.PiecesRoot] + if filePieceHashes == nil { + filePieceHashes = make([][32]byte, file.numPieces()) + generics.MakeMapIfNil(&pc.receivedHashPieces) + pc.receivedHashPieces[msg.PiecesRoot] = filePieceHashes + } + if msg.ProofLayers != 0 { + // This isn't handled yet. + panic(msg.ProofLayers) + } + copy(filePieceHashes[msg.Index:], msg.Hashes) + root := merkle.RootWithPadHash( + filePieceHashes, + metainfo.HashForPiecePad(int64(pc.t.usualPieceSize()))) + expectedPiecesRoot := file.piecesRoot.Unwrap() + if root == expectedPiecesRoot { + pc.logger.WithNames(v2HashesLogName).Levelf( + log.Info, + "got piece hashes for file %v (num pieces %v)", + file, file.numPieces()) + for filePieceIndex, peerHash := range filePieceHashes { + torrentPieceIndex := file.BeginPieceIndex() + filePieceIndex + pc.t.piece(torrentPieceIndex).hashV2.Set(peerHash) + } + } else { + pc.logger.WithNames(v2HashesLogName).Levelf( + log.Debug, + "peer file piece hashes root mismatch: %x != %x", + root, expectedPiecesRoot) + } + return nil +} + type hashRequest struct { piecesRoot [32]byte baseLayer, index, length, proofLayers pp.Integer diff --git a/torrent.go b/torrent.go index 8a403e16ab..e7b579b3c0 100644 --- a/torrent.go +++ b/torrent.go @@ -2006,8 +2006,7 @@ func (t *Torrent) AnnounceToDht(s DhtServer) (done <-chan struct{}, stop func(), go func() { defer stop() defer close(_done) - // Won't this race? - err = eg.Wait() + eg.Wait() }() return } @@ -3118,3 +3117,12 @@ func (t *Torrent) eachShortInfohash(each func(short [20]byte)) { each(*t.infoHashV2.Value.ToShort()) } } + +func (t *Torrent) getFileByPiecesRoot(hash [32]byte) *File { + for _, f := range *t.files { + if f.piecesRoot.Unwrap() == hash { + return f + } + } + return nil +} diff --git a/v2hashes.go b/v2hashes.go index 10cbafc73d..0525daa066 100644 --- a/v2hashes.go +++ b/v2hashes.go @@ -1 +1,5 @@ package torrent + +const ( + v2HashesLogName = "v2hashes" +) From 38c4ce1b1d345382d7812a9a4aab3e5c55e5f44e Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 1 Mar 2024 23:40:05 +1100 Subject: [PATCH 12/15] Misc improvements --- cmd/torrent2/main.go | 21 +++++++++++++++++++++ merkle/hash.go | 9 +++++++-- metainfo/file-tree.go | 20 ++++++++++++++++---- metainfo/fileinfo.go | 3 ++- metainfo/info.go | 2 +- metainfo/piece.go | 2 +- segments/segments.go | 14 +++----------- spec.go | 8 ++++---- torrent.go | 6 +++++- 9 files changed, 60 insertions(+), 25 deletions(-) diff --git a/cmd/torrent2/main.go b/cmd/torrent2/main.go index 0d57f0133b..6017464ff2 100644 --- a/cmd/torrent2/main.go +++ b/cmd/torrent2/main.go @@ -4,6 +4,7 @@ package main import ( + "fmt" "os" "github.com/anacrolix/torrent/metainfo" @@ -39,6 +40,26 @@ func main() { err = metainfo.ValidatePieceLayers(mi.PieceLayers, &info.FileTree, info.PieceLength) assertOk(err) }, + "pprint": func() { + mi, err := metainfo.LoadFromFile(args[2]) + assertOk(err) + info, err := mi.UnmarshalInfo() + assertOk(err) + files := info.UpvertedFiles() + pieceIndex := 0 + for _, f := range files { + numPieces := int((f.Length + info.PieceLength - 1) / info.PieceLength) + endIndex := pieceIndex + numPieces + fmt.Printf( + "%x: %q: pieces (%v-%v)\n", + f.PiecesRoot.Unwrap(), + f.BestPath(), + pieceIndex, + endIndex-1, + ) + pieceIndex = endIndex + } + }, }[args[1]]() }, }[args[0]]() diff --git a/merkle/hash.go b/merkle/hash.go index 1a8f67a922..5984547aab 100644 --- a/merkle/hash.go +++ b/merkle/hash.go @@ -3,12 +3,14 @@ package merkle import ( "crypto/sha256" "hash" + "unsafe" ) func NewHash() *Hash { - return &Hash{ + h := &Hash{ nextBlock: sha256.New(), } + return h } type Hash struct { @@ -41,7 +43,9 @@ func (h *Hash) Write(p []byte) (n int, err error) { } func (h *Hash) nextBlockSum() (sum [32]byte) { - h.nextBlock.Sum(sum[:0]) + if unsafe.SliceData(h.nextBlock.Sum(sum[:0])) != unsafe.SliceData(sum[:]) { + panic("go sux") + } return } @@ -57,6 +61,7 @@ func (h *Hash) Sum(b []byte) []byte { func (h *Hash) Reset() { h.blocks = h.blocks[:0] h.nextBlock.Reset() + h.written = 0 } func (h *Hash) Size() int { diff --git a/metainfo/file-tree.go b/metainfo/file-tree.go index bfb7229e9a..0d7e1caf3b 100644 --- a/metainfo/file-tree.go +++ b/metainfo/file-tree.go @@ -64,23 +64,35 @@ func (ft *FileTree) orderedKeys() []string { return keys } -func (ft *FileTree) UpvertedFiles(path []string, out func(fi FileInfo)) { +func (ft *FileTree) upvertedFiles(pieceLength int64, out func(fi FileInfo)) { + var offset int64 + ft.upvertedFilesInner(pieceLength, nil, &offset, out) +} + +func (ft *FileTree) upvertedFilesInner( + pieceLength int64, + path []string, + offset *int64, + out func(fi FileInfo), +) { if ft.IsDir() { for _, key := range ft.orderedKeys() { if key == FileTreePropertiesKey { continue } sub := g.MapMustGet(ft.Dir, key) - sub.UpvertedFiles(append(path, key), out) + sub.upvertedFilesInner(pieceLength, append(path, key), offset, out) } } else { out(FileInfo{ Length: ft.File.Length, Path: append([]string(nil), path...), // BEP 52 requires paths be UTF-8 if possible. - PathUtf8: append([]string(nil), path...), - PiecesRoot: ft.PiecesRootAsByteArray(), + PathUtf8: append([]string(nil), path...), + PiecesRoot: ft.PiecesRootAsByteArray(), + TorrentOffset: *offset, }) + *offset += (ft.File.Length + pieceLength - 1) / pieceLength * pieceLength } } diff --git a/metainfo/fileinfo.go b/metainfo/fileinfo.go index 66ee2da8b8..25c262529b 100644 --- a/metainfo/fileinfo.go +++ b/metainfo/fileinfo.go @@ -18,7 +18,8 @@ type FileInfo struct { // BEP 52. This isn't encoded in a v1 FileInfo, but is exposed here for APIs that expect to deal // v1 files. - PiecesRoot g.Option[[32]byte] `bencode:"-"` + PiecesRoot g.Option[[32]byte] `bencode:"-"` + TorrentOffset int64 `bencode:"-"` } func (fi *FileInfo) DisplayPath(info *Info) string { diff --git a/metainfo/info.go b/metainfo/info.go index d1b54d44d6..8b8749c7c1 100644 --- a/metainfo/info.go +++ b/metainfo/info.go @@ -155,7 +155,7 @@ func (info *Info) IsDir() bool { // single and multi-file torrent infos. func (info *Info) UpvertedFiles() (files []FileInfo) { if info.HasV2() { - info.FileTree.UpvertedFiles(nil, func(fi FileInfo) { + info.FileTree.upvertedFiles(info.PieceLength, func(fi FileInfo) { files = append(files, fi) }) return diff --git a/metainfo/piece.go b/metainfo/piece.go index c7377f5db6..4972e52926 100644 --- a/metainfo/piece.go +++ b/metainfo/piece.go @@ -13,7 +13,7 @@ func (p Piece) Length() int64 { pieceLength := p.Info.PieceLength lastFileEnd := int64(0) done := false - p.Info.FileTree.UpvertedFiles(nil, func(fi FileInfo) { + p.Info.FileTree.upvertedFiles(pieceLength, func(fi FileInfo) { if done { return } diff --git a/segments/segments.go b/segments/segments.go index 90e77ce0d7..3870042589 100644 --- a/segments/segments.go +++ b/segments/segments.go @@ -4,16 +4,6 @@ type Int = int64 type Length = Int -func min(i Int, rest ...Int) Int { - ret := i - for _, i := range rest { - if i < ret { - ret = i - } - } - return ret -} - type Extent struct { Start, Length Int } @@ -23,10 +13,12 @@ func (e Extent) End() Int { } type ( - Callback = func(int, Extent) bool + Callback = func(segmentIndex int, segmentBounds Extent) bool LengthIter = func() (Length, bool) ) +// Returns true if callback returns false early, or all segments in the haystack for the needle are +// found. func Scan(haystack LengthIter, needle Extent, callback Callback) bool { i := 0 for needle.Length != 0 { diff --git a/spec.go b/spec.go index 49a1104ede..f54b325b5c 100644 --- a/spec.go +++ b/spec.go @@ -71,13 +71,13 @@ func TorrentSpecFromMetaInfoErr(mi *metainfo.MetaInfo) (*TorrentSpec, error) { if err != nil { err = fmt.Errorf("unmarshalling info: %w", err) } - v1Ih := mi.HashInfoBytes() + var v1Ih metainfo.Hash + if info.HasV1() { + v1Ih = mi.HashInfoBytes() + } var v2Infohash g.Option[infohash_v2.T] if info.HasV2() { v2Infohash.Set(infohash_v2.HashBytes(mi.InfoBytes)) - if !info.HasV1() { - v1Ih = *v2Infohash.Value.ToShort() - } } return &TorrentSpec{ diff --git a/torrent.go b/torrent.go index e7b579b3c0..64e8066012 100644 --- a/torrent.go +++ b/torrent.go @@ -2489,11 +2489,15 @@ func (t *Torrent) dropBannedPeers() { func (t *Torrent) pieceHasher(index pieceIndex) { p := t.piece(index) + // Do we really need to spell out that it's a copy error? If it's a failure to hash the hash + // will just be wrong. correct, failedPeers, copyErr := t.hashPiece(index) switch copyErr { case nil, io.EOF: default: - log.Fmsg("piece %v (%s) hash failure copy error: %v", p, p.hash.HexString(), copyErr).Log(t.logger) + t.logger.Levelf( + log.Warning, + "error hashing piece %v: %v", index, copyErr) } t.storageLock.RUnlock() t.cl.lock() From c9235820da1d6ffa5dcfebfa4c6d9697140f1db3 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Fri, 1 Mar 2024 23:54:56 +1100 Subject: [PATCH 13/15] Fix file storage segments for v2 torrents --- common/upverted_files.go | 9 +++++++++ segments/index.go | 6 ++++++ storage/file-piece.go | 16 ++++++++++++---- storage/file.go | 2 +- torrent.go | 6 +++++- webseed/client.go | 2 +- 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/common/upverted_files.go b/common/upverted_files.go index 1933e16a13..8984eb251c 100644 --- a/common/upverted_files.go +++ b/common/upverted_files.go @@ -16,3 +16,12 @@ func LengthIterFromUpvertedFiles(fis []metainfo.FileInfo) segments.LengthIter { return l, true } } + +// Returns file segments, BitTorrent v2 aware. +func TorrentOffsetFileSegments(info *metainfo.Info) (ret []segments.Extent) { + files := info.UpvertedFiles() + for _, fi := range files { + ret = append(ret, segments.Extent{fi.TorrentOffset, fi.Length}) + } + return +} diff --git a/segments/index.go b/segments/index.go index 6717dcba46..888e90a813 100644 --- a/segments/index.go +++ b/segments/index.go @@ -17,6 +17,10 @@ type Index struct { segments []Extent } +func NewIndexFromSegments(segments []Extent) Index { + return Index{segments} +} + func (me Index) iterSegments() func() (Length, bool) { return func() (Length, bool) { if len(me.segments) == 0 { @@ -29,6 +33,8 @@ func (me Index) iterSegments() func() (Length, bool) { } } +// Returns true if the callback returns false early, or extents are found in the index for all parts +// of the given extent. func (me Index) Locate(e Extent, output Callback) bool { first := sort.Search(len(me.segments), func(i int) bool { _e := me.segments[i] diff --git a/storage/file-piece.go b/storage/file-piece.go index 47772017e3..99ff5fc867 100644 --- a/storage/file-piece.go +++ b/storage/file-piece.go @@ -1,6 +1,7 @@ package storage import ( + "github.com/anacrolix/torrent/segments" "io" "log" "os" @@ -32,12 +33,19 @@ func (fs *filePieceImpl) Completion() Completion { verified := true if c.Complete { // If it's allegedly complete, check that its constituent files have the necessary length. - for _, fi := range extentCompleteRequiredLengths(fs.p.Info, fs.p.Offset(), fs.p.Length()) { - s, err := os.Stat(fs.files[fi.fileIndex].path) - if err != nil || s.Size() < fi.length { + if !fs.segmentLocater.Locate(segments.Extent{ + Start: fs.p.Offset(), + Length: fs.p.Length(), + }, func(i int, extent segments.Extent) bool { + file := fs.files[i] + s, err := os.Stat(file.path) + if err != nil || s.Size() < extent.Start+extent.Length { verified = false - break + return false } + return true + }) { + panic("files do not cover piece extent") } } diff --git a/storage/file.go b/storage/file.go index b873964787..231825fdfe 100644 --- a/storage/file.go +++ b/storage/file.go @@ -83,7 +83,7 @@ func (fs fileClientImpl) OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash } t := &fileTorrentImpl{ files, - segments.NewIndex(common.LengthIterFromUpvertedFiles(upvertedFiles)), + segments.NewIndexFromSegments(common.TorrentOffsetFileSegments(info)), infoHash, fs.opts.PieceCompletion, } diff --git a/torrent.go b/torrent.go index 64e8066012..a361c4edd6 100644 --- a/torrent.go +++ b/torrent.go @@ -1133,7 +1133,11 @@ func (t *Torrent) hashPiece(piece pieceIndex) ( if logPieceContents { writers = append(writers, &examineBuf) } - _, err = storagePiece.WriteTo(io.MultiWriter(writers...)) + var written int64 + written, err = storagePiece.WriteTo(io.MultiWriter(writers...)) + if err == nil && written != int64(p.length()) { + err = io.ErrShortWrite + } if logPieceContents { t.logger.WithDefaultLevel(log.Debug).Printf("hashed %q with copy err %v", examineBuf.Bytes(), err) } diff --git a/webseed/client.go b/webseed/client.go index d5ae3ac1db..4614a3e407 100644 --- a/webseed/client.go +++ b/webseed/client.go @@ -64,7 +64,7 @@ func (me *Client) SetInfo(info *metainfo.Info) { // http://ia600500.us.archive.org/1/items URLs in archive.org torrents. return } - me.fileIndex = segments.NewIndex(common.LengthIterFromUpvertedFiles(info.UpvertedFiles())) + me.fileIndex = segments.NewIndexFromSegments(common.TorrentOffsetFileSegments(info)) me.info = info me.Pieces.AddRange(0, uint64(info.NumPieces())) } From 188f5bb314d333803ba0ce37f94dc7451ecdd9e1 Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Sat, 2 Mar 2024 13:00:28 +1100 Subject: [PATCH 14/15] Improve segment handling for discontiguous extents --- metainfo/file-tree.go | 15 ++++---- metainfo/info.go | 8 ++++- segments/index.go | 28 ++++++++++----- segments/segments.go | 45 +++++++++++++++-------- segments/segments_test.go | 14 +++++--- storage/file-misc.go | 46 +++++++++--------------- storage/file-misc_test.go | 75 +++++++++++++++++++++++++++++++++++---- 7 files changed, 159 insertions(+), 72 deletions(-) diff --git a/metainfo/file-tree.go b/metainfo/file-tree.go index 0d7e1caf3b..2b651bd446 100644 --- a/metainfo/file-tree.go +++ b/metainfo/file-tree.go @@ -11,12 +11,15 @@ import ( const FileTreePropertiesKey = "" +type FileTreeFile struct { + Length int64 `bencode:"length"` + PiecesRoot string `bencode:"pieces root"` +} + +// The fields here don't need bencode tags as the marshalling is done manually. type FileTree struct { - File struct { - Length int64 `bencode:"length"` - PiecesRoot string `bencode:"pieces root"` - } - Dir map[string]FileTree + File FileTreeFile + Dir map[string]FileTree } func (ft *FileTree) UnmarshalBencode(bytes []byte) (err error) { @@ -107,7 +110,7 @@ func (ft *FileTree) Walk(path []string, f func(path []string, ft *FileTree)) { } func (ft *FileTree) PiecesRootAsByteArray() (ret g.Option[[32]byte]) { - if ft.File.Length == 0 { + if ft.File.PiecesRoot == "" { return } n := copy(ret.Value[:], ft.File.PiecesRoot) diff --git a/metainfo/info.go b/metainfo/info.go index 8b8749c7c1..5d6300ec51 100644 --- a/metainfo/info.go +++ b/metainfo/info.go @@ -168,7 +168,13 @@ func (info *Info) UpvertedFiles() (files []FileInfo) { Path: nil, }} } - return info.Files + var offset int64 + for _, fi := range info.Files { + fi.TorrentOffset = offset + offset += fi.Length + files = append(files, fi) + } + return } func (info *Info) Piece(index int) Piece { diff --git a/segments/index.go b/segments/index.go index 888e90a813..d7ae744b12 100644 --- a/segments/index.go +++ b/segments/index.go @@ -1,6 +1,7 @@ package segments import ( + g "github.com/anacrolix/generics" "sort" ) @@ -21,15 +22,19 @@ func NewIndexFromSegments(segments []Extent) Index { return Index{segments} } -func (me Index) iterSegments() func() (Length, bool) { - return func() (Length, bool) { +func (me Index) iterSegments() func() (Extent, bool) { + var lastEnd g.Option[Int] + return func() (ret Extent, ok bool) { if len(me.segments) == 0 { - return 0, false - } else { - l := me.segments[0].Length - me.segments = me.segments[1:] - return l, true + return } + cur := me.segments[0] + me.segments = me.segments[1:] + ret.Start = cur.Start - lastEnd.UnwrapOr(cur.Start) + ret.Length = cur.Length + lastEnd.Set(cur.End()) + ok = true + return } } @@ -41,11 +46,16 @@ func (me Index) Locate(e Extent, output Callback) bool { return _e.End() > e.Start }) if first == len(me.segments) { - return false + return e.Length == 0 } e.Start -= me.segments[first].Start + // The extent is before the first segment. + if e.Start < 0 { + e.Length += e.Start + e.Start = 0 + } me.segments = me.segments[first:] - return Scan(me.iterSegments(), e, func(i int, e Extent) bool { + return ScanConsecutive(me.iterSegments(), e, func(i int, e Extent) bool { return output(i+first, e) }) } diff --git a/segments/segments.go b/segments/segments.go index 3870042589..35e7e4c8e5 100644 --- a/segments/segments.go +++ b/segments/segments.go @@ -13,33 +13,48 @@ func (e Extent) End() Int { } type ( - Callback = func(segmentIndex int, segmentBounds Extent) bool - LengthIter = func() (Length, bool) + Callback = func(segmentIndex int, segmentBounds Extent) bool + LengthIter = func() (Length, bool) + ConsecutiveExtentIter = func() (Extent, bool) ) // Returns true if callback returns false early, or all segments in the haystack for the needle are // found. func Scan(haystack LengthIter, needle Extent, callback Callback) bool { + return ScanConsecutive( + func() (Extent, bool) { + l, ok := haystack() + return Extent{0, l}, ok + }, + needle, + callback, + ) +} + +// Returns true if callback returns false early, or all segments in the haystack for the needle are +// found. +func ScanConsecutive(haystack ConsecutiveExtentIter, needle Extent, callback Callback) bool { i := 0 + // Extents have been found in the haystack and we're waiting for the needle to end. This is kind + // of for backwards compatibility for some tests that expect to have zero-length extents. + startedNeedle := false for needle.Length != 0 { l, ok := haystack() if !ok { return false } - if needle.Start < l || needle.Start == l && l == 0 { - e1 := Extent{ - Start: needle.Start, - Length: min(l, needle.End()) - needle.Start, - } - if e1.Length >= 0 { - if !callback(i, e1) { - return true - } - needle.Start = 0 - needle.Length -= e1.Length + + e1 := Extent{ + Start: max(needle.Start-l.Start, 0), + } + e1.Length = max(min(l.Length, needle.End()-l.Start)-e1.Start, 0) + needle.Start = max(0, needle.Start-l.End()) + needle.Length -= e1.Length + l.Start + if e1.Length > 0 || (startedNeedle && needle.Length != 0) { + if !callback(i, e1) { + return true } - } else { - needle.Start -= l + startedNeedle = true } i++ } diff --git a/segments/segments_test.go b/segments/segments_test.go index 9ce9164bf7..e757654f9d 100644 --- a/segments/segments_test.go +++ b/segments/segments_test.go @@ -1,9 +1,8 @@ package segments import ( + qt "github.com/frankban/quicktest" "testing" - - "github.com/stretchr/testify/assert" ) func LengthIterFromSlice(ls []Length) LengthIter { @@ -36,14 +35,21 @@ func (me *collectExtents) scanCallback(i int, e Extent) bool { type newLocater func(LengthIter) Locater -func assertLocate(t *testing.T, nl newLocater, ls []Length, needle Extent, firstExpectedIndex int, expectedExtents []Extent) { +func assertLocate( + t *testing.T, + nl newLocater, + ls []Length, + needle Extent, + firstExpectedIndex int, + expectedExtents []Extent, +) { var actual collectExtents var expected collectExtents for i, e := range expectedExtents { expected.scanCallback(firstExpectedIndex+i, e) } nl(LengthIterFromSlice(ls))(needle, actual.scanCallback) - assert.EqualValues(t, expected, actual) + qt.Check(t, actual, qt.DeepEquals, expected) } func testLocater(t *testing.T, newLocater newLocater) { diff --git a/storage/file-misc.go b/storage/file-misc.go index 8966ecbd1a..67514420f0 100644 --- a/storage/file-misc.go +++ b/storage/file-misc.go @@ -1,34 +1,20 @@ package storage -import "github.com/anacrolix/torrent/metainfo" +import ( + "github.com/anacrolix/torrent/segments" +) -type requiredLength struct { - fileIndex int - length int64 -} - -func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) { - if n == 0 { - return - } - for i, fi := range info.UpvertedFiles() { - if off >= fi.Length { - off -= fi.Length - continue - } - n1 := n - if off+n1 > fi.Length { - n1 = fi.Length - off - } - ret = append(ret, requiredLength{ - fileIndex: i, - length: off + n1, - }) - n -= n1 - if n == 0 { - return - } - off = 0 - } - panic("extent exceeds torrent bounds") +// Returns the minimum file lengths required for the given extent to exist on disk. Returns false if +// the extent is not covered by the files in the index. +func minFileLengthsForTorrentExtent( + fileSegmentsIndex segments.Index, + off, n int64, + each func(fileIndex int, length int64) bool, +) bool { + return fileSegmentsIndex.Locate(segments.Extent{ + Start: off, + Length: n, + }, func(fileIndex int, segmentBounds segments.Extent) bool { + return each(fileIndex, segmentBounds.Start+segmentBounds.Length) + }) } diff --git a/storage/file-misc_test.go b/storage/file-misc_test.go index f74196d047..9e97ce00ec 100644 --- a/storage/file-misc_test.go +++ b/storage/file-misc_test.go @@ -1,6 +1,9 @@ package storage import ( + "github.com/anacrolix/torrent/common" + "github.com/anacrolix/torrent/segments" + qt "github.com/frankban/quicktest" "testing" "github.com/stretchr/testify/assert" @@ -8,6 +11,60 @@ import ( "github.com/anacrolix/torrent/metainfo" ) +type requiredLength struct { + FileIndex int + Length int64 +} + +// The required file indices and file lengths for the given extent to be "complete". This is the +// outdated interface used by some tests. +func extentCompleteRequiredLengths(info *metainfo.Info, off, n int64) (ret []requiredLength) { + index := segments.NewIndexFromSegments(common.TorrentOffsetFileSegments(info)) + minFileLengthsForTorrentExtent(index, off, n, func(fileIndex int, length int64) bool { + ret = append(ret, requiredLength{fileIndex, length}) + return true + }) + return +} + +func TestExtentCompleteRequiredLengthsV2InfoWithGaps(t *testing.T) { + info := &metainfo.Info{ + MetaVersion: 2, + PieceLength: 2, + FileTree: metainfo.FileTree{ + Dir: map[string]metainfo.FileTree{ + "a": { + File: metainfo.FileTreeFile{ + Length: 2, + }, + }, + "b": { + File: metainfo.FileTreeFile{Length: 3}, + }, + // Here there's a gap where v2 torrents piece align, so the next file offset starts + // at 6. + "c": { + File: metainfo.FileTreeFile{Length: 4}, + }, + }, + }, + } + c := qt.New(t) + check := func(off, n int64, expected ...requiredLength) { + c.Check(extentCompleteRequiredLengths(info, off, n), qt.DeepEquals, expected) + } + check(0, 0) + check(0, 1, requiredLength{FileIndex: 0, Length: 1}) + check(0, 2, requiredLength{FileIndex: 0, Length: 2}) + check(0, 3, requiredLength{FileIndex: 0, Length: 2}, requiredLength{FileIndex: 1, Length: 1}) + check(2, 2, requiredLength{FileIndex: 1, Length: 2}) + check(4, 1, requiredLength{FileIndex: 1, Length: 3}) + check(5, 0) + check(4, 2, requiredLength{FileIndex: 1, Length: 3}) + check(5, 1) + check(6, 4, requiredLength{FileIndex: 2, Length: 4}) +} + func TestExtentCompleteRequiredLengths(t *testing.T) { info := &metainfo.Info{ Files: []metainfo.FileInfo{ @@ -15,23 +72,27 @@ func TestExtentCompleteRequiredLengths(t *testing.T) { {Path: []string{"b"}, Length: 3}, }, } + c := qt.New(t) + check := func(off, n int64, expected ...requiredLength) { + c.Check(extentCompleteRequiredLengths(info, off, n), qt.DeepEquals, expected) + } assert.Empty(t, extentCompleteRequiredLengths(info, 0, 0)) assert.EqualValues(t, []requiredLength{ - {fileIndex: 0, length: 1}, + {FileIndex: 0, Length: 1}, }, extentCompleteRequiredLengths(info, 0, 1)) assert.EqualValues(t, []requiredLength{ - {fileIndex: 0, length: 2}, + {FileIndex: 0, Length: 2}, }, extentCompleteRequiredLengths(info, 0, 2)) assert.EqualValues(t, []requiredLength{ - {fileIndex: 0, length: 2}, - {fileIndex: 1, length: 1}, + {FileIndex: 0, Length: 2}, + {FileIndex: 1, Length: 1}, }, extentCompleteRequiredLengths(info, 0, 3)) assert.EqualValues(t, []requiredLength{ - {fileIndex: 1, length: 2}, + {FileIndex: 1, Length: 2}, }, extentCompleteRequiredLengths(info, 2, 2)) assert.EqualValues(t, []requiredLength{ - {fileIndex: 1, length: 3}, + {FileIndex: 1, Length: 3}, }, extentCompleteRequiredLengths(info, 4, 1)) assert.Len(t, extentCompleteRequiredLengths(info, 5, 0), 0) - assert.Panics(t, func() { extentCompleteRequiredLengths(info, 6, 1) }) + check(6, 1) } From 026bbe2a6f52b056099caeb865769ee4137cd3dd Mon Sep 17 00:00:00 2001 From: Matt Joiner Date: Mon, 4 Mar 2024 16:01:22 +1100 Subject: [PATCH 15/15] Use UpvertedFiles instead of raw Info.Files in a few places --- cmd/torrent/serve.go | 2 +- fs/torrentfs.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/torrent/serve.go b/cmd/torrent/serve.go index bdb1559473..d37dafafaa 100644 --- a/cmd/torrent/serve.go +++ b/cmd/torrent/serve.go @@ -44,7 +44,7 @@ func serve() (cmd bargle.Command) { if err != nil { return fmt.Errorf("building info from path %q: %w", filePath, err) } - for _, fi := range info.Files { + for _, fi := range info.UpvertedFiles() { log.Printf("added %q", fi.Path) } mi := metainfo.MetaInfo{ diff --git a/fs/torrentfs.go b/fs/torrentfs.go index 5e0b75ea18..1b7b144e78 100644 --- a/fs/torrentfs.go +++ b/fs/torrentfs.go @@ -72,7 +72,7 @@ func isSubPath(parent, child string) bool { func (dn dirNode) ReadDirAll(ctx context.Context) (des []fuse.Dirent, err error) { names := map[string]bool{} - for _, fi := range dn.metadata.Files { + for _, fi := range dn.metadata.UpvertedFiles() { filePathname := strings.Join(fi.Path, "/") if !isSubPath(dn.path, filePathname) { continue