From d1f654148f98772352f9faa21dbc7fb85614ee84 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 Nov 2024 18:03:42 +0100 Subject: [PATCH 01/16] fix: increase provider sample size (#10589) --- core/node/provider.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/node/provider.go b/core/node/provider.go index 7f37cd8b1a4..d6ab26b0c0d 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -14,8 +14,12 @@ import ( "go.uber.org/fx" ) +// The size of a batch that will be used for calculating average announcement +// time per CID, inside of boxo/provider.ThroughputReport +// and in 'ipfs stats provide' report. +const sampledBatchSize = 1000 + func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { - const magicThroughputReportCount = 128 return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { opts := []provider.Option{ provider.Online(cr), @@ -105,7 +109,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count), reprovideInterval) } return false - }, magicThroughputReportCount)) + }, sampledBatchSize)) } sys, err := provider.New(repo.Datastore(), opts...) if err != nil { From 0bd0edc782401f1bed59c5696dbc441abbeeff85 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 Nov 2024 18:04:53 +0100 Subject: [PATCH 02/16] feat(bootstrap): add JS-based va1.bootstrap.libp2p.io (#10575) * feat(bootstrap): /dnsaddr/va1.bootstrap.libp2p.io this adds https://github.com/libp2p/js-libp2p-amino-dht-bootstrapper instance at /dnsaddr/va1.bootstrap.libp2p.io to ensure Kubo does not depend on single language stack for initial bootstrap * docs: document where defaults live * test: updated bootstrappers --- config/bootstrap_peers.go | 7 ++++--- docs/config.md | 2 +- test/sharness/t0120-bootstrap.sh | 9 ++++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config/bootstrap_peers.go b/config/bootstrap_peers.go index 1671d9f8156..91f8ea1afa4 100644 --- a/config/bootstrap_peers.go +++ b/config/bootstrap_peers.go @@ -16,11 +16,12 @@ import ( // import dependency issue. TODO: move this into a config/default/ package. var DefaultBootstrapAddresses = []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", // rust-libp2p-server "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", - "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io - "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io + "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", // js-libp2p-amino-dht-bootstrapper + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io + "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io } // ErrInvalidPeerAddr signals an address is not a valid peer address. diff --git a/docs/config.md b/docs/config.md index d2610b088fa..a745fec01e5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -567,7 +567,7 @@ Type: `optionalString` Bootstrap is an array of [multiaddrs][multiaddr] of trusted nodes that your node connects to, to fetch other nodes of the network on startup. -Default: The ipfs.io bootstrap nodes +Default: [`config.DefaultBootstrapAddresses`](https://github.com/ipfs/kubo/blob/master/config/bootstrap_peers.go) Type: `array[string]` ([multiaddrs][multiaddr]) diff --git a/test/sharness/t0120-bootstrap.sh b/test/sharness/t0120-bootstrap.sh index 2922533c628..00141da1f06 100755 --- a/test/sharness/t0120-bootstrap.sh +++ b/test/sharness/t0120-bootstrap.sh @@ -9,8 +9,9 @@ BP1="/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTez BP2="/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa" BP3="/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" BP4="/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" -BP5="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" -BP6="/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" +BP5="/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8" +BP6="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" +BP7="/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" test_description="Test ipfs repo operations" @@ -93,10 +94,11 @@ test_bootstrap_cmd() { echo "added $BP4" >>add2_expected && echo "added $BP5" >>add2_expected && echo "added $BP6" >>add2_expected && + echo "added $BP7" >>add2_expected && test_cmp add2_expected add2_actual ' - test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 $BP6 + test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 $BP6 $BP7 test_expect_success "'ipfs bootstrap rm --all' succeeds" ' ipfs bootstrap rm --all >rm2_actual @@ -109,6 +111,7 @@ test_bootstrap_cmd() { echo "removed $BP4" >>rm2_expected && echo "removed $BP5" >>rm2_expected && echo "removed $BP6" >>rm2_expected && + echo "removed $BP7" >>rm2_expected && test_cmp rm2_expected rm2_actual ' From ef58568964b9f4a399fe508a77ba305a5ca6d383 Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Mon, 25 Nov 2024 13:15:43 -1000 Subject: [PATCH 03/16] fix some typos in docs (#10598) --- docs/add-code-flow.md | 4 ++-- docs/datastores.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/add-code-flow.md b/docs/add-code-flow.md index a13c7177d40..264731a3b13 100644 --- a/docs/add-code-flow.md +++ b/docs/add-code-flow.md @@ -1,6 +1,6 @@ # IPFS : The `Add` command demystified -The goal of this document is to capture the code flow for adding a file (see the `coreapi` package) using the IPFS CLI, in the process exploring some datastructures and packages like `ipld.Node` (aka `dagnode`), `FSNode`, `MFS`, etc. +The goal of this document is to capture the code flow for adding a file (see the `coreapi` package) using the IPFS CLI, in the process exploring some data structures and packages like `ipld.Node` (aka `dagnode`), `FSNode`, `MFS`, etc. ## Concepts - [Files](https://github.com/ipfs/docs/issues/133) @@ -99,4 +99,4 @@ Within the function, a new `Adder` is created with the configured `Blockstore` a - **[`adder.PinRoot()`](https://github.com/ipfs/go-ipfs/blob/v0.4.18/core/coreunix/add.go#L171)** - *Pin all files under the `MFS` **root*** - The whole process ends with `PinRoot` recursively pinning all the files under the `MFS` **root** \ No newline at end of file + The whole process ends with `PinRoot` recursively pinning all the files under the `MFS` **root** diff --git a/docs/datastores.md b/docs/datastores.md index f2051b601d4..03a400cb6aa 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -53,7 +53,7 @@ Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store. } ``` -The following options are availble for tuning pebble. +The following options are available for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. * `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) From 37c5060742368cfc37a0da53c6c2ad500109365e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 26 Nov 2024 12:34:06 +0100 Subject: [PATCH 04/16] Update to boxo with refactored providerQueryManager. (#10595) --- core/commands/bitswap.go | 21 +------ core/core.go | 4 +- core/coreapi/pin.go | 2 +- core/coreapi/unixfs.go | 2 +- core/node/bitswap.go | 63 ++++++++++++++++--- core/node/groups.go | 5 +- docs/changelogs/v0.32.md | 2 +- docs/changelogs/v0.33.md | 2 + docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- .../delegated_routing_v1_http_proxy_test.go | 6 +- .../delegated_routing_v1_http_server_test.go | 4 ++ test/cli/provider_test.go | 3 + test/cli/routing_dht_test.go | 5 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 +- test/sharness/t0220-bitswap.sh | 3 - 19 files changed, 93 insertions(+), 47 deletions(-) diff --git a/core/commands/bitswap.go b/core/commands/bitswap.go index 07f91fb0f52..7f7ab963651 100644 --- a/core/commands/bitswap.go +++ b/core/commands/bitswap.go @@ -5,7 +5,6 @@ import ( "io" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" - e "github.com/ipfs/kubo/core/commands/e" humanize "github.com/dustin/go-humanize" bitswap "github.com/ipfs/boxo/bitswap" @@ -53,10 +52,7 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`, return ErrNotOnline } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } + bs := nd.Bitswap pstr, found := req.Options[peerOptionName].(string) if found { @@ -112,12 +108,7 @@ var bitswapStatCmd = &cmds.Command{ return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline) } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } - - st, err := bs.Stat() + st, err := nd.Bitswap.Stat() if err != nil { return err } @@ -134,7 +125,6 @@ var bitswapStatCmd = &cmds.Command{ human, _ := req.Options[bitswapHumanOptionName].(bool) fmt.Fprintln(w, "bitswap status") - fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize) fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived) fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent) if human { @@ -190,17 +180,12 @@ prints the ledger associated with a given peer. return ErrNotOnline } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } - partner, err := peer.Decode(req.Arguments[0]) if err != nil { return err } - return cmds.EmitOnce(res, bs.LedgerForPeer(partner)) + return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner)) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error { diff --git a/core/core.go b/core/core.go index 0c9333e0683..54c98752762 100644 --- a/core/core.go +++ b/core/core.go @@ -19,6 +19,7 @@ import ( pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/go-datastore" + bitswap "github.com/ipfs/boxo/bitswap" bserv "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" @@ -102,7 +103,8 @@ type IpfsNode struct { UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks - Exchange exchange.Interface // the block exchange + strategy (bitswap) + Exchange exchange.Interface // the block exchange + strategy + Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance Namesys namesys.NameSystem // the name system, resolves paths to hashes Provider provider.System // the value provider system IpnsRepub *ipnsrp.Republisher `optional:"true"` diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 22b3aa25c24..64c65b651c2 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -44,7 +44,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp return fmt.Errorf("pin: %s", err) } - if err := api.provider.Provide(dagNode.Cid()); err != nil { + if err := api.provider.Provide(ctx, dagNode.Cid(), true); err != nil { return err } diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index e175488f37f..dbeeefda426 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -173,7 +173,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options } if !settings.OnlyHash { - if err := api.provider.Provide(nd.Cid()); err != nil { + if err := api.provider.Provide(ctx, nd.Cid(), true); err != nil { return path.ImmutablePath{}, err } } diff --git a/core/node/bitswap.go b/core/node/bitswap.go index 4132d5a0186..d379a5866ce 100644 --- a/core/node/bitswap.go +++ b/core/node/bitswap.go @@ -5,9 +5,12 @@ import ( "time" "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/client" "github.com/ipfs/boxo/bitswap/network" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" + "github.com/ipfs/boxo/exchange/providing" + provider "github.com/ipfs/boxo/provider" "github.com/ipfs/kubo/config" irouting "github.com/ipfs/kubo/routing" "github.com/libp2p/go-libp2p/core/host" @@ -34,7 +37,7 @@ type bitswapOptionsOut struct { // BitswapOptions creates configuration options for Bitswap from the config file // and whether to provide data. -func BitswapOptions(cfg *config.Config, provide bool) interface{} { +func BitswapOptions(cfg *config.Config) interface{} { return func() bitswapOptionsOut { var internalBsCfg config.InternalBitswap if cfg.Internal.Bitswap != nil { @@ -42,7 +45,6 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} { } opts := []bitswap.Option{ - bitswap.ProvideEnabled(provide), bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))), bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))), @@ -55,7 +57,7 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} { } } -type onlineExchangeIn struct { +type bitswapIn struct { fx.In Mctx helpers.MetricsCtx @@ -65,19 +67,62 @@ type onlineExchangeIn struct { BitswapOpts []bitswap.Option `group:"bitswap-options"` } -// OnlineExchange creates new LibP2P backed block exchange (BitSwap). +// Bitswap creates the BitSwap server/client instance. // Additional options to bitswap.New can be provided via the "bitswap-options" // group. -func OnlineExchange() interface{} { - return func(in onlineExchangeIn, lc fx.Lifecycle) exchange.Interface { - bitswapNetwork := network.NewFromIpfsHost(in.Host, in.Rt) +func Bitswap(provide bool) interface{} { + return func(in bitswapIn, lc fx.Lifecycle) *bitswap.Bitswap { + bitswapNetwork := network.NewFromIpfsHost(in.Host) + + var provider client.ProviderFinder + if provide { + provider = in.Rt + } + bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, in.Bs, in.BitswapOpts...) - exch := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, in.Bs, in.BitswapOpts...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - return exch.Close() + return bs.Close() }, }) + return bs + } +} + +// OnlineExchange creates new LibP2P backed block exchange. +func OnlineExchange() interface{} { + return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface { + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return in.Close() + }, + }) + return in + } +} + +type providingExchangeIn struct { + fx.In + + BaseExch exchange.Interface + Provider provider.System +} + +// ProvidingExchange creates a providing.Exchange with the existing exchange +// and the provider.System. +// We cannot do this in OnlineExchange because it causes cycles so this is for +// a decorator. +func ProvidingExchange(provide bool) interface{} { + return func(in providingExchangeIn, lc fx.Lifecycle) exchange.Interface { + exch := in.BaseExch + if provide { + exch = providing.New(in.BaseExch, in.Provider) + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return exch.Close() + }, + }) + } return exch } } diff --git a/core/node/groups.go b/core/node/groups.go index c74d0be8a2c..519cbb47d6e 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -293,8 +293,11 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part shouldBitswapProvide := !cfg.Experimental.StrategicProviding return fx.Options( - fx.Provide(BitswapOptions(cfg, shouldBitswapProvide)), + fx.Provide(BitswapOptions(cfg)), + fx.Provide(Bitswap(shouldBitswapProvide)), fx.Provide(OnlineExchange()), + // Replace our Exchange with a Providing exchange! + fx.Decorate(ProvidingExchange(shouldBitswapProvide)), fx.Provide(DNSResolver), fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))), fx.Provide(Peering), diff --git a/docs/changelogs/v0.32.md b/docs/changelogs/v0.32.md index 790390978ee..f00cca611a7 100644 --- a/docs/changelogs/v0.32.md +++ b/docs/changelogs/v0.32.md @@ -1,6 +1,6 @@ # Kubo changelog v0.32 -- [v0.32.0](#v0310) +- [v0.32.0](#v0320) ## v0.32.0 diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index 711092a2cfd..df5f9158e96 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -16,6 +16,8 @@ #### ๐Ÿ“ฆ๏ธ Dependency updates +- update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index d3b7aee834a..e3968887772 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.23 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.37.0 github.com/multiformats/go-multiaddr v0.13.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f4da6f07adc..f601c797d02 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index 5de417a990e..5ea2e63dad1 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index abf6d83d9b7..f27573fd484 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/cli/delegated_routing_v1_http_proxy_test.go b/test/cli/delegated_routing_v1_http_proxy_test.go index 1d80ae50a5f..79fd726a411 100644 --- a/test/cli/delegated_routing_v1_http_proxy_test.go +++ b/test/cli/delegated_routing_v1_http_proxy_test.go @@ -60,8 +60,10 @@ func TestRoutingV1Proxy(t *testing.T) { nodes := setupNodes(t) cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000)) - - res := nodes[1].IPFS("routing", "findprovs", cidStr) + // Reprovide as initialProviderDelay still ongoing + res := nodes[0].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) + res = nodes[1].IPFS("routing", "findprovs", cidStr) assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed()) }) diff --git a/test/cli/delegated_routing_v1_http_server_test.go b/test/cli/delegated_routing_v1_http_server_test.go index f2bd98cb77b..916b9188280 100644 --- a/test/cli/delegated_routing_v1_http_server_test.go +++ b/test/cli/delegated_routing_v1_http_server_test.go @@ -14,6 +14,7 @@ import ( "github.com/ipfs/kubo/test/cli/harness" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRoutingV1Server(t *testing.T) { @@ -38,6 +39,9 @@ func TestRoutingV1Server(t *testing.T) { text := "hello world " + uuid.New().String() cidStr := nodes[2].IPFSAddStr(text) _ = nodes[3].IPFSAddStr(text) + // Reprovide as initialProviderDelay still ongoing + res := nodes[3].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) cid, err := cid.Decode(cidStr) assert.NoError(t, err) diff --git a/test/cli/provider_test.go b/test/cli/provider_test.go index 5ecf8f3cab7..546ac3fd711 100644 --- a/test/cli/provider_test.go +++ b/test/cli/provider_test.go @@ -42,6 +42,9 @@ func TestProvider(t *testing.T) { defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) + // Reprovide as initialProviderDelay still ongoing + res := nodes[0].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) diff --git a/test/cli/routing_dht_test.go b/test/cli/routing_dht_test.go index fb0d391951e..d149e93a2a3 100644 --- a/test/cli/routing_dht_test.go +++ b/test/cli/routing_dht_test.go @@ -84,7 +84,10 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) { t.Run("ipfs routing findprovs", func(t *testing.T) { t.Parallel() hash := nodes[3].IPFSAddStr("some stuff") - res := nodes[4].IPFS("routing", "findprovs", hash) + // Reprovide as initialProviderDelay still ongoing + res := nodes[3].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) + res = nodes[4].IPFS("routing", "findprovs", hash) assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed()) }) diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 73840e0d045..3c2c9c7873a 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -119,7 +119,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f // indirect + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 84672271d22..f980982a4ab 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -318,8 +318,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y= -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= diff --git a/test/sharness/t0220-bitswap.sh b/test/sharness/t0220-bitswap.sh index 3575f0d33e2..412437651be 100755 --- a/test/sharness/t0220-bitswap.sh +++ b/test/sharness/t0220-bitswap.sh @@ -18,7 +18,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 @@ -56,7 +55,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 @@ -85,7 +83,6 @@ test_expect_success "'ipfs bitswap stat --human' succeeds" ' test_expect_success "'ipfs bitswap stat --human' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 B From fb58b0ae85f43c938c50a55aa1f88d72c41c6eb3 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 26 Nov 2024 19:19:07 +0100 Subject: [PATCH 05/16] tests/cli: fix flapping tests (#10600) The issue is that for dht providing to work correctly, at least two nodes must have dht routing. Before, dht providing in the test always failed, but the test succeeded when the tested-CID was the one that the failure happened with (the first in the list for ProvideMany). --- test/cli/delegated_routing_v1_http_proxy_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/cli/delegated_routing_v1_http_proxy_test.go b/test/cli/delegated_routing_v1_http_proxy_test.go index 79fd726a411..ef3bc4fe2bc 100644 --- a/test/cli/delegated_routing_v1_http_proxy_test.go +++ b/test/cli/delegated_routing_v1_http_proxy_test.go @@ -15,9 +15,11 @@ func TestRoutingV1Proxy(t *testing.T) { t.Parallel() setupNodes := func(t *testing.T) harness.Nodes { - nodes := harness.NewT(t).NewNodes(2).Init() + nodes := harness.NewT(t).NewNodes(3).Init() - // Node 0 uses DHT and exposes the Routing API. + // Node 0 uses DHT and exposes the Routing API. For the DHT + // to actually work there will need to be another DHT-enabled + // node. nodes[0].UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Discovery.MDNS.Enabled = false @@ -49,6 +51,15 @@ func TestRoutingV1Proxy(t *testing.T) { }) nodes[1].StartDaemon() + // This is the second DHT node. Only used so that the DHT is + // operative. + nodes[2].UpdateConfig(func(cfg *config.Config) { + cfg.Gateway.ExposeRoutingAPI = config.True + cfg.Discovery.MDNS.Enabled = false + cfg.Routing.Type = config.NewOptionalString("dht") + }) + nodes[2].StartDaemon() + // Connect them. nodes.Connect() From 466f72f518bd9853035d973718ac7bddb8df0281 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 26 Nov 2024 19:19:48 +0100 Subject: [PATCH 06/16] docs: typos (#10602) --- docs/config.md | 8 ++++---- docs/datastores.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/config.md b/docs/config.md index a745fec01e5..08a1cceedd8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -496,15 +496,15 @@ Enables AutoTLS feature to get DNS+TLS for [libp2p Secure WebSocket](https://git If `.../tls/sni/*.libp2p.direct/ws` [multiaddr] is present in [`Addresses.Swarm`](#addressesswarm) with SNI segment ending with [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix), -Kubo will obtain and set up a trusted PKI TLS certificate for it, making it diallable from web browser's [Secure Contexts](https://w3c.github.io/webappsec-secure-contexts/). +Kubo will obtain and set up a trusted PKI TLS certificate for it, making it dialable from web browser's [Secure Contexts](https://w3c.github.io/webappsec-secure-contexts/). > [!IMPORTANT] > Caveats: -> - Requires your Kubo node to be publicly diallable. +> - Requires your Kubo node to be publicly dialable. > - If you want to test this with a node that is behind a NAT and uses manual port forwarding or UPnP (`Swarm.DisableNatPortMap=false`), > add catch-all `/ip4/0.0.0.0/tcp/4002/tls/sni/*.libp2p.direct/ws` and `/ip6/::/tcp/4002/tls/sni/*.libp2p.direct/ws` to [`Addresses.Swarm`](#addressesswarm) > and **wait 5-15 minutes** for libp2p node to set up and learn about own public addresses via [AutoNAT](#autonat). -> - If your node is fresh and just started, the [p2p-forge] client may produce and log ERRORs during this time, but once a publicly diallable addresses are set up, a subsequent retry should be successful. +> - If your node is fresh and just started, the [p2p-forge] client may produce and log ERRORs during this time, but once a publicly dialable addresses are set up, a subsequent retry should be successful. > - Listeners defined in [`Addresses.Swarm`](#addressesswarm) with `/tls/sni` must use a separate port from other TCP listeners, e.g. `4002` instead of the default `4001`. > - A separate port (`/tcp/4002`) has to be used instead of `/tcp/4001` because we wait for TCP port sharing ([go-libp2p#2984](https://github.com/libp2p/go-libp2p/issues/2684)) to be implemented. > - If you use manual port forwarding, make sure incoming connections to this additional port are allowed the same way `4001` ones already are. @@ -536,7 +536,7 @@ Do not change this unless you self-host [p2p-forge] under own domain. > [!IMPORTANT] > The default endpoint performs [libp2p Peer ID Authentication over HTTP](https://github.com/libp2p/specs/blob/master/http/peer-id-auth.md) > (proving ownership of PeerID), probes if your Kubo node can correctly answer to a [libp2p Identify](https://github.com/libp2p/specs/tree/master/identify) query. -> This ensures only a correctly configured, publicly diallable Kubo can initiate [ACME DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for `peerid.libp2p.direct`. +> This ensures only a correctly configured, publicly dialable Kubo can initiate [ACME DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for `peerid.libp2p.direct`. Default: `https://registration.libp2p.direct` (public good run by [Interplanetary Shipyard](https://ipshipyard.com)) diff --git a/docs/datastores.md b/docs/datastores.md index 03a400cb6aa..321f59bfc8d 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -57,7 +57,7 @@ The following options are available for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. * `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) -* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) +* `disableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) * `cacheSize`: Size of pebble's shared block cache. (default: 8MB) * `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction. * `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels. From 23ef1d70e775a4e6bff09e834133a3c4efec19d6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 26 Nov 2024 19:49:02 +0100 Subject: [PATCH 07/16] fix: go-libp2p v0.37.2 (#10603) * chore: go-libp2p v0.37.1 https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1 * chore: go-libp2p v0.37.2 https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2 --- docs/changelogs/v0.33.md | 1 + docs/examples/kubo-as-a-library/go.mod | 8 ++++---- docs/examples/kubo-as-a-library/go.sum | 16 ++++++++-------- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- test/dependencies/go.mod | 8 ++++---- test/dependencies/go.sum | 16 ++++++++-------- 7 files changed, 37 insertions(+), 36 deletions(-) diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index df5f9158e96..0daf81b0814 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -17,6 +17,7 @@ #### ๐Ÿ“ฆ๏ธ Dependency updates - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) +- update `go-libp2p` to [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) ### ๐Ÿ“ Changelog diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index e3968887772..0ec843bd551 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.37.0 + github.com/libp2p/go-libp2p v0.37.2 github.com/multiformats/go-multiaddr v0.13.0 ) @@ -147,12 +147,12 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect + github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect - github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-multistream v0.6.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.20.2 // indirect @@ -185,7 +185,7 @@ require ( github.com/prometheus/common v0.60.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.48.1 // indirect + github.com/quic-go/quic-go v0.48.2 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index f601c797d02..fe15d3d6dd4 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -459,8 +459,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.37.0 h1:8K3mcZgwTldydMCNOiNi/ZJrOB9BY+GlI3UxYzxBi9A= -github.com/libp2p/go-libp2p v0.37.0/go.mod h1:GOKmSN99scDuYGTwaTbQPR8Nt6dxrK3ue7OjW2NGDg4= +github.com/libp2p/go-libp2p v0.37.2 h1:Irh+n9aDPTLt9wJYwtlHu6AhMUipbC1cGoJtOiBqI9c= +github.com/libp2p/go-libp2p v0.37.2/go.mod h1:M8CRRywYkqC6xKHdZ45hmqVckBj5z4mRLIMLWReypz8= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= @@ -555,8 +555,8 @@ github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y9 github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= -github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= -github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= @@ -574,8 +574,8 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= -github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-multistream v0.6.0 h1:ZaHKbsL404720283o4c/IHQXiS6gb8qAN5EIJ4PN5EA= +github.com/multiformats/go-multistream v0.6.0/go.mod h1:MOyoG5otO24cHIg8kf9QW2/NozURlkP/rvi2FQJyCPg= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -689,8 +689,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= -github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= diff --git a/go.mod b/go.mod index 5ea2e63dad1..5b7c7a544f0 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.37.0 + github.com/libp2p/go-libp2p v0.37.2 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.28.1 github.com/libp2p/go-libp2p-kbucket v0.6.4 @@ -65,7 +65,7 @@ require ( github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-socket-activation v0.1.0 github.com/multiformats/go-multiaddr v0.13.0 - github.com/multiformats/go-multiaddr-dns v0.4.0 + github.com/multiformats/go-multiaddr-dns v0.4.1 github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 @@ -193,7 +193,7 @@ require ( github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-multistream v0.6.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.20.2 // indirect @@ -224,7 +224,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.26.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.48.1 // indirect + github.com/quic-go/quic-go v0.48.2 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index f27573fd484..5a3faa4f2c1 100644 --- a/go.sum +++ b/go.sum @@ -539,8 +539,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.37.0 h1:8K3mcZgwTldydMCNOiNi/ZJrOB9BY+GlI3UxYzxBi9A= -github.com/libp2p/go-libp2p v0.37.0/go.mod h1:GOKmSN99scDuYGTwaTbQPR8Nt6dxrK3ue7OjW2NGDg4= +github.com/libp2p/go-libp2p v0.37.2 h1:Irh+n9aDPTLt9wJYwtlHu6AhMUipbC1cGoJtOiBqI9c= +github.com/libp2p/go-libp2p v0.37.2/go.mod h1:M8CRRywYkqC6xKHdZ45hmqVckBj5z4mRLIMLWReypz8= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= @@ -655,8 +655,8 @@ github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/o github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= -github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= -github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= @@ -674,8 +674,8 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= -github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-multistream v0.6.0 h1:ZaHKbsL404720283o4c/IHQXiS6gb8qAN5EIJ4PN5EA= +github.com/multiformats/go-multistream v0.6.0/go.mod h1:MOyoG5otO24cHIg8kf9QW2/NozURlkP/rvi2FQJyCPg= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -816,8 +816,8 @@ github.com/prometheus/statsd_exporter v0.26.1 h1:ucbIAdPmwAUcA+dU+Opok8Qt81Aw8Ha github.com/prometheus/statsd_exporter v0.26.1/go.mod h1:XlDdjAmRmx3JVvPPYuFNUg+Ynyb5kR69iPPkQjxXFMk= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= -github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 3c2c9c7873a..4d9d535a407 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -156,7 +156,7 @@ require ( github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect - github.com/libp2p/go-libp2p v0.37.0 // indirect + github.com/libp2p/go-libp2p v0.37.2 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect @@ -185,11 +185,11 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect + github.com/multiformats/go-multiaddr-dns v0.4.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect - github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-multistream v0.6.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nakabonne/nestif v0.3.1 // indirect @@ -229,7 +229,7 @@ require ( github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.48.1 // indirect + github.com/quic-go/quic-go v0.48.2 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index f980982a4ab..bc73c85dfe8 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -431,8 +431,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.2.0 h1:EIZzjmeOE6c8Dav0sNv35vhZxATIXWZg6j/C08XmmDw= github.com/libp2p/go-flow-metrics v0.2.0/go.mod h1:st3qqfu8+pMfh+9Mzqb2GTiwrAGjIPszEjZmtksN8Jc= -github.com/libp2p/go-libp2p v0.37.0 h1:8K3mcZgwTldydMCNOiNi/ZJrOB9BY+GlI3UxYzxBi9A= -github.com/libp2p/go-libp2p v0.37.0/go.mod h1:GOKmSN99scDuYGTwaTbQPR8Nt6dxrK3ue7OjW2NGDg4= +github.com/libp2p/go-libp2p v0.37.2 h1:Irh+n9aDPTLt9wJYwtlHu6AhMUipbC1cGoJtOiBqI9c= +github.com/libp2p/go-libp2p v0.37.2/go.mod h1:M8CRRywYkqC6xKHdZ45hmqVckBj5z4mRLIMLWReypz8= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= @@ -520,8 +520,8 @@ github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= -github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= -github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= +github.com/multiformats/go-multiaddr-dns v0.4.1 h1:whi/uCLbDS3mSEUMb1MsoT4uzUeZB0N32yzufqS0i5M= +github.com/multiformats/go-multiaddr-dns v0.4.1/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= @@ -531,8 +531,8 @@ github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI1 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= -github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-multistream v0.6.0 h1:ZaHKbsL404720283o4c/IHQXiS6gb8qAN5EIJ4PN5EA= +github.com/multiformats/go-multistream v0.6.0/go.mod h1:MOyoG5otO24cHIg8kf9QW2/NozURlkP/rvi2FQJyCPg= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -646,8 +646,8 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4l github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= -github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg= github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= From 3b2c2698e7ac06b91e8a08ebd85b581155474f52 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 27 Nov 2024 19:28:11 +0100 Subject: [PATCH 08/16] chore: p2p-forge/client v0.1.0 (#10605) --- docs/changelogs/v0.33.md | 1 + docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 12 ++++++------ go.mod | 6 +++--- go.sum | 16 ++++++++-------- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index 0daf81b0814..e73182174bf 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -18,6 +18,7 @@ - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) - update `go-libp2p` to [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) +- update `p2p-forge/client` to [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0) ### ๐Ÿ“ Changelog diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 0ec843bd551..3c025040ccf 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -109,7 +109,7 @@ require ( github.com/ipld/go-car/v2 v2.14.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect - github.com/ipshipyard/p2p-forge v0.0.2 // indirect + github.com/ipshipyard/p2p-forge v0.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index fe15d3d6dd4..110c9bf8e17 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -401,8 +401,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -730,8 +730,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= -github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= +github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= +github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1074,8 +1074,8 @@ golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/go.mod b/go.mod index 5b7c7a544f0..a9bca9d7f33 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/ipld/go-car/v2 v2.14.2 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.21.0 - github.com/ipshipyard/p2p-forge v0.0.2 + github.com/ipshipyard/p2p-forge v0.1.0 github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 @@ -222,7 +222,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/statsd_exporter v0.26.1 // indirect + github.com/prometheus/statsd_exporter v0.27.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect @@ -231,7 +231,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/rs/cors v1.11.1 // indirect github.com/samber/lo v1.47.0 // indirect - github.com/slok/go-http-metrics v0.12.0 // indirect + github.com/slok/go-http-metrics v0.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 5a3faa4f2c1..c29dbd816bf 100644 --- a/go.sum +++ b/go.sum @@ -469,8 +469,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= @@ -812,8 +812,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/prometheus/statsd_exporter v0.26.1 h1:ucbIAdPmwAUcA+dU+Opok8Qt81Aw8HanlO+2N/Wjv7w= -github.com/prometheus/statsd_exporter v0.26.1/go.mod h1:XlDdjAmRmx3JVvPPYuFNUg+Ynyb5kR69iPPkQjxXFMk= +github.com/prometheus/statsd_exporter v0.27.1 h1:tcRJOmwlA83HPfWzosAgr2+zEN5XDFv+M2mn/uYkn5Y= +github.com/prometheus/statsd_exporter v0.27.1/go.mod h1:vA6ryDfsN7py/3JApEst6nLTJboq66XsNcJGNmC88NQ= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= @@ -864,8 +864,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= -github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= +github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= +github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1288,8 +1288,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 4d9d535a407..93ab3f923e0 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -130,7 +130,7 @@ require ( github.com/ipfs/kubo v0.31.0 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect - github.com/ipshipyard/p2p-forge v0.0.2 // indirect + github.com/ipshipyard/p2p-forge v0.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index bc73c85dfe8..11be0c149fd 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -358,8 +358,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= From 02995d5d7dd99250198003d9cb4c09dd6193c4b7 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 27 Nov 2024 19:35:15 +0100 Subject: [PATCH 09/16] tests(cli/harness): use unused Verbose flag to pipe daemon outputs (#10601) Obtain actual daemon output on the screen while debugging tests. Might be useful in the future even though it needs to manually set verbose somewhere. --- test/cli/harness/run.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/cli/harness/run.go b/test/cli/harness/run.go index 8ca85eb63b1..077af6ca574 100644 --- a/test/cli/harness/run.go +++ b/test/cli/harness/run.go @@ -3,6 +3,7 @@ package harness import ( "fmt" "io" + "os" "os/exec" "strings" ) @@ -60,8 +61,27 @@ func environToMap(environ []string) map[string]string { func (r *Runner) Run(req RunRequest) *RunResult { cmd := exec.Command(req.Path, req.Args...) - stdout := &Buffer{} - stderr := &Buffer{} + var stdout io.Writer + var stderr io.Writer + outbuf := &Buffer{} + errbuf := &Buffer{} + + if r.Verbose { + or, ow := io.Pipe() + errr, errw := io.Pipe() + stdout = io.MultiWriter(outbuf, ow) + stderr = io.MultiWriter(errbuf, errw) + go func() { + _, _ = io.Copy(os.Stdout, or) + }() + go func() { + _, _ = io.Copy(os.Stderr, errr) + }() + } else { + stdout = outbuf + stderr = errbuf + } + cmd.Stdout = stdout cmd.Stderr = stderr cmd.Dir = r.Dir @@ -83,8 +103,8 @@ func (r *Runner) Run(req RunRequest) *RunResult { err := req.RunFunc(cmd) result := RunResult{ - Stdout: stdout, - Stderr: stderr, + Stdout: outbuf, + Stderr: errbuf, Cmd: cmd, Err: err, } From 9433105a85d3acdbc12cd26d593f62b618b55767 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 27 Nov 2024 19:40:16 +0100 Subject: [PATCH 10/16] docs(changelog/v0.33): bitswap reprovide changes (#10604) --- docs/changelogs/v0.33.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index e73182174bf..1230fca37ce 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -14,6 +14,10 @@ ### ๐Ÿ”ฆ Highlights +#### Bitswap improvements from Boxo + +This release includes some refactorings and improvements affecting Bitswap which should improve reliability. One of the changes affects blocks providing. Previously, the bitswap layer took care itself of announcing new blocks -added or received- with the configured provider (i.e. DHT). This bypassed the "Reprovider", that is, the system that manages precisely "providing" the blocks stored by Kubo. The Reprovider knows how to take advantage of the [AcceleratedDHTClient](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), is able to handle priorities, logs statistics and is able to resume on daemon reboot where it left off. From now on, Bitswap will not be doing any providing on-the-side and all announcements are managed by the reprovider. In some cases, when the reproviding queue is full with other elements, this may cause additional delays, but more likely this will result in improved block-providing behaviour overall. + #### ๐Ÿ“ฆ๏ธ Dependency updates - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) From fa769f7544896969ae58b811804e265f8b1e02f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 19:41:55 +0100 Subject: [PATCH 11/16] chore(deps): bump codecov/codecov-action from 4.6.0 to 5.0.7 (#10597) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.6.0 to 5.0.7. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238...015f24e6818733317a2da2edd6290ab26238649a) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/gotest.yml | 2 +- .github/workflows/sharness.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index c6cac2c78de..c3846b03fab 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -45,7 +45,7 @@ jobs: make -j "$PARALLEL" test/unit/gotest.junit.xml && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7 if: failure() || success() with: name: unittests diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index ac91b132161..354cad1029e 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -55,7 +55,7 @@ jobs: # increasing parallelism beyond 10 doesn't speed up the tests much PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }} - name: Upload coverage report - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7 if: failure() || success() with: name: sharness From 778a4189528f252bd3e5fb1257d53ff1119d2926 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Thu, 28 Nov 2024 03:50:43 +0800 Subject: [PATCH 12/16] refactor(rcmgr): use default libp2p rcmgr metrics (#9947) Co-authored-by: Marcin Rataj --- core/node/libp2p/rcmgr.go | 9 +- core/node/libp2p/rcmgr_metrics.go | 251 ------------------ docs/changelogs/v0.33.md | 6 + ...prometheus_metrics_added_by_enabling_rcmgr | 4 - 4 files changed, 10 insertions(+), 260 deletions(-) delete mode 100644 core/node/libp2p/rcmgr_metrics.go diff --git a/core/node/libp2p/rcmgr.go b/core/node/libp2p/rcmgr.go index 80bfec34ae6..977461e04bd 100644 --- a/core/node/libp2p/rcmgr.go +++ b/core/node/libp2p/rcmgr.go @@ -7,6 +7,10 @@ import ( "os" "path/filepath" + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core/node/helpers" + "github.com/ipfs/kubo/repo" + "github.com/benbjohnson/clock" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" @@ -16,10 +20,6 @@ import ( rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/multiformats/go-multiaddr" "go.uber.org/fx" - - "github.com/ipfs/kubo/config" - "github.com/ipfs/kubo/core/node/helpers" - "github.com/ipfs/kubo/repo" ) var rcmgrLogger = logging.Logger("rcmgr") @@ -70,7 +70,6 @@ filled in with autocomputed defaults.`) } ropts := []rcmgr.Option{ - rcmgr.WithMetrics(createRcmgrMetrics()), rcmgr.WithTraceReporter(str), rcmgr.WithLimitPerSubnet( nil, diff --git a/core/node/libp2p/rcmgr_metrics.go b/core/node/libp2p/rcmgr_metrics.go deleted file mode 100644 index f8b1a7daa3b..00000000000 --- a/core/node/libp2p/rcmgr_metrics.go +++ /dev/null @@ -1,251 +0,0 @@ -package libp2p - -import ( - "errors" - "strconv" - - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" - rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" - - "github.com/prometheus/client_golang/prometheus" -) - -func mustRegister(c prometheus.Collector) { - err := prometheus.Register(c) - are := prometheus.AlreadyRegisteredError{} - if errors.As(err, &are) { - return - } - if err != nil { - panic(err) - } -} - -func createRcmgrMetrics() rcmgr.MetricsReporter { - const ( - direction = "direction" - usesFD = "usesFD" - protocol = "protocol" - service = "service" - ) - - connAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_conns_allowed_total", - Help: "allowed connections", - }, - []string{direction, usesFD}, - ) - mustRegister(connAllowed) - - connBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_conns_blocked_total", - Help: "blocked connections", - }, - []string{direction, usesFD}, - ) - mustRegister(connBlocked) - - streamAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_streams_allowed_total", - Help: "allowed streams", - }, - []string{direction}, - ) - mustRegister(streamAllowed) - - streamBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_streams_blocked_total", - Help: "blocked streams", - }, - []string{direction}, - ) - mustRegister(streamBlocked) - - peerAllowed := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_peers_allowed_total", - Help: "allowed peers", - }) - mustRegister(peerAllowed) - - peerBlocked := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_peer_blocked_total", - Help: "blocked peers", - }) - mustRegister(peerBlocked) - - protocolAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_allowed_total", - Help: "allowed streams attached to a protocol", - }, - []string{protocol}, - ) - mustRegister(protocolAllowed) - - protocolBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_blocked_total", - Help: "blocked streams attached to a protocol", - }, - []string{protocol}, - ) - mustRegister(protocolBlocked) - - protocolPeerBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_for_peer_blocked_total", - Help: "blocked streams attached to a protocol for a specific peer", - }, - []string{protocol}, - ) - mustRegister(protocolPeerBlocked) - - serviceAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_services_allowed_total", - Help: "allowed streams attached to a service", - }, - []string{service}, - ) - mustRegister(serviceAllowed) - - serviceBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_services_blocked_total", - Help: "blocked streams attached to a service", - }, - []string{service}, - ) - mustRegister(serviceBlocked) - - servicePeerBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_service_for_peer_blocked_total", - Help: "blocked streams attached to a service for a specific peer", - }, - []string{service}, - ) - mustRegister(servicePeerBlocked) - - memoryAllowed := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_memory_allocations_allowed_total", - Help: "allowed memory allocations", - }) - mustRegister(memoryAllowed) - - memoryBlocked := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_memory_allocations_blocked_total", - Help: "blocked memory allocations", - }) - mustRegister(memoryBlocked) - - return rcmgrMetrics{ - connAllowed, - connBlocked, - streamAllowed, - streamBlocked, - peerAllowed, - peerBlocked, - protocolAllowed, - protocolBlocked, - protocolPeerBlocked, - serviceAllowed, - serviceBlocked, - servicePeerBlocked, - memoryAllowed, - memoryBlocked, - } -} - -// Failsafe to ensure interface from go-libp2p-resource-manager is implemented -var _ rcmgr.MetricsReporter = rcmgrMetrics{} - -type rcmgrMetrics struct { - connAllowed *prometheus.CounterVec - connBlocked *prometheus.CounterVec - streamAllowed *prometheus.CounterVec - streamBlocked *prometheus.CounterVec - peerAllowed prometheus.Counter - peerBlocked prometheus.Counter - protocolAllowed *prometheus.CounterVec - protocolBlocked *prometheus.CounterVec - protocolPeerBlocked *prometheus.CounterVec - serviceAllowed *prometheus.CounterVec - serviceBlocked *prometheus.CounterVec - servicePeerBlocked *prometheus.CounterVec - memoryAllowed prometheus.Counter - memoryBlocked prometheus.Counter -} - -func getDirection(d network.Direction) string { - switch d { - default: - return "" - case network.DirInbound: - return "inbound" - case network.DirOutbound: - return "outbound" - } -} - -func (r rcmgrMetrics) AllowConn(dir network.Direction, usefd bool) { - r.connAllowed.WithLabelValues(getDirection(dir), strconv.FormatBool(usefd)).Inc() -} - -func (r rcmgrMetrics) BlockConn(dir network.Direction, usefd bool) { - r.connBlocked.WithLabelValues(getDirection(dir), strconv.FormatBool(usefd)).Inc() -} - -func (r rcmgrMetrics) AllowStream(_ peer.ID, dir network.Direction) { - r.streamAllowed.WithLabelValues(getDirection(dir)).Inc() -} - -func (r rcmgrMetrics) BlockStream(_ peer.ID, dir network.Direction) { - r.streamBlocked.WithLabelValues(getDirection(dir)).Inc() -} - -func (r rcmgrMetrics) AllowPeer(_ peer.ID) { - r.peerAllowed.Inc() -} - -func (r rcmgrMetrics) BlockPeer(_ peer.ID) { - r.peerBlocked.Inc() -} - -func (r rcmgrMetrics) AllowProtocol(proto protocol.ID) { - r.protocolAllowed.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) BlockProtocol(proto protocol.ID) { - r.protocolBlocked.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) BlockProtocolPeer(proto protocol.ID, _ peer.ID) { - r.protocolPeerBlocked.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) AllowService(svc string) { - r.serviceAllowed.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) BlockService(svc string) { - r.serviceBlocked.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) BlockServicePeer(svc string, _ peer.ID) { - r.servicePeerBlocked.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) AllowMemory(_ int) { - r.memoryAllowed.Inc() -} - -func (r rcmgrMetrics) BlockMemory(_ int) { - r.memoryBlocked.Inc() -} diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index 1230fca37ce..cd0d0f7db21 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -18,6 +18,12 @@ This release includes some refactorings and improvements affecting Bitswap which should improve reliability. One of the changes affects blocks providing. Previously, the bitswap layer took care itself of announcing new blocks -added or received- with the configured provider (i.e. DHT). This bypassed the "Reprovider", that is, the system that manages precisely "providing" the blocks stored by Kubo. The Reprovider knows how to take advantage of the [AcceleratedDHTClient](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), is able to handle priorities, logs statistics and is able to resume on daemon reboot where it left off. From now on, Bitswap will not be doing any providing on-the-side and all announcements are managed by the reprovider. In some cases, when the reproviding queue is full with other elements, this may cause additional delays, but more likely this will result in improved block-providing behaviour overall. +#### Using default `libp2p_rcmgr` metrics + +Bespoke rcmgr metrics [were removed](https://github.com/ipfs/kubo/pull/9947), Kubo now exposes only the default `libp2p_rcmgr` metrics from go-libp2p. +This makes it easier to compare Kubo with custom implementations based on go-libp2p. +If you depended on removed ones, please fill an issue to add them to the upstream [go-libp2p](https://github.com/libp2p/go-libp2p). + #### ๐Ÿ“ฆ๏ธ Dependency updates - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) diff --git a/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr b/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr index 382ab125602..e69de29bb2d 100644 --- a/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr +++ b/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr @@ -1,4 +0,0 @@ -libp2p_rcmgr_memory_allocations_allowed_total -libp2p_rcmgr_memory_allocations_blocked_total -libp2p_rcmgr_peer_blocked_total -libp2p_rcmgr_peers_allowed_total From 8654538cce527e5e7c30ca69ec522a1eec5ab7f5 Mon Sep 17 00:00:00 2001 From: John <153272819+hishope@users.noreply.github.com> Date: Sat, 30 Nov 2024 03:23:13 +0800 Subject: [PATCH 13/16] chore: fix broken url in comment (#10606) Signed-off-by: hishope --- tracing/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracing/doc.go b/tracing/doc.go index d442ea2db50..2c9711a63a1 100644 --- a/tracing/doc.go +++ b/tracing/doc.go @@ -6,7 +6,7 @@ // // Tracing is configured through environment variables, as consistent with the OpenTelemetry spec as possible: // -// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md // // OTEL_TRACES_EXPORTER: a comma-separated list of exporters: // - otlp From 53e793afdd2c457296d59ffa7d5ed342cf695787 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 3 Dec 2024 20:11:34 +0100 Subject: [PATCH 14/16] fix: ipfs-webui v4.4.1 (#10608) https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1 --- core/corehttp/webui.go | 3 ++- docs/changelogs/v0.33.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index 4f00ad71202..f0c4c3c1cff 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // WebUI version confirmed to work with this Kubo version -const WebUIPath = "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm" // v4.4.0 +const WebUIPath = "/ipfs/bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4" // v4.4.1 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm", "/ipfs/bafybeid4uxz7klxcu3ffsnmn64r7ihvysamlj4ohl5h2orjsffuegcpaeq", "/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy", "/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4", diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index cd0d0f7db21..a5d8ef2ef34 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -29,6 +29,7 @@ If you depended on removed ones, please fill an issue to add them to the upstrea - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) - update `go-libp2p` to [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) - update `p2p-forge/client` to [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0) +- update `ipfs-webui` to [v4.4.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1) ### ๐Ÿ“ Changelog From 224d6a3ba4fbf690312cb436cf78b4cac3fb7ec6 Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:15:33 -1000 Subject: [PATCH 15/16] refactor(cmds): do not return errors embedded in result type (#10527) incl. https://github.com/ipfs/boxo/pull/738 --- client/rpc/pin.go | 64 ++++------- client/rpc/unixfs.go | 116 ++++++++----------- cmd/ipfs/kubo/pinmfs.go | 2 +- core/commands/ls.go | 23 ++-- core/commands/pin/pin.go | 18 +-- core/commands/pin/remotepin.go | 69 +++++------ core/coreapi/pin.go | 153 ++++++++++++------------- core/coreapi/unixfs.go | 78 ++++++++----- core/coreiface/pin.go | 8 +- core/coreiface/tests/block.go | 12 +- core/coreiface/tests/pin.go | 46 ++++---- core/coreiface/tests/unixfs.go | 68 +++++++---- core/coreiface/unixfs.go | 58 +++++++++- docs/examples/kubo-as-a-library/go.mod | 4 +- docs/examples/kubo-as-a-library/go.sum | 8 +- go.mod | 4 +- go.sum | 8 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 8 +- 19 files changed, 408 insertions(+), 341 deletions(-) diff --git a/client/rpc/pin.go b/client/rpc/pin.go index 6e8e942ac0e..2b1ef825801 100644 --- a/client/rpc/pin.go +++ b/client/rpc/pin.go @@ -62,10 +62,12 @@ type pinLsObject struct { Type string } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, pins chan<- iface.Pin, opts ...caopts.PinLsOption) error { + defer close(pins) + options, err := caopts.PinLsOptions(opts...) if err != nil { - return nil, err + return err } res, err := api.core().Request("pin/ls"). @@ -73,48 +75,32 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan i Option("stream", true). Send(ctx) if err != nil { - return nil, err + return err } - - pins := make(chan iface.Pin) - go func(ch chan<- iface.Pin) { - defer res.Output.Close() - defer close(ch) - - dec := json.NewDecoder(res.Output) - var out pinLsObject - for { - switch err := dec.Decode(&out); err { - case nil: - case io.EOF: - return - default: - select { - case ch <- pin{err: err}: - return - case <-ctx.Done(): - return - } + defer res.Output.Close() + + dec := json.NewDecoder(res.Output) + var out pinLsObject + for { + err := dec.Decode(&out) + if err != nil { + if err != io.EOF { + return err } + return nil + } - c, err := cid.Parse(out.Cid) - if err != nil { - select { - case ch <- pin{err: err}: - return - case <-ctx.Done(): - return - } - } + c, err := cid.Parse(out.Cid) + if err != nil { + return err + } - select { - case ch <- pin{typ: out.Type, name: out.Name, path: path.FromCid(c)}: - case <-ctx.Done(): - return - } + select { + case pins <- pin{typ: out.Type, name: out.Name, path: path.FromCid(c)}: + case <-ctx.Done(): + return ctx.Err() } - }(pins) - return pins, nil + } } // IsPinned returns whether or not the given cid is pinned diff --git a/client/rpc/unixfs.go b/client/rpc/unixfs.go index 3ba2c1c15f0..316cc21a8ec 100644 --- a/client/rpc/unixfs.go +++ b/client/rpc/unixfs.go @@ -144,10 +144,12 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- iface.DirEntry, opts ...caopts.UnixfsLsOption) error { + defer close(out) + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { - return nil, err + return err } resp, err := api.core().Request("ls", p.String()). @@ -156,86 +158,64 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.Unixfs Option("stream", true). Send(ctx) if err != nil { - return nil, err + return err } if resp.Error != nil { - return nil, resp.Error + return err } + defer resp.Close() dec := json.NewDecoder(resp.Output) - out := make(chan iface.DirEntry) - - go func() { - defer resp.Close() - defer close(out) - - for { - var link lsOutput - if err := dec.Decode(&link); err != nil { - if err == io.EOF { - return - } - select { - case out <- iface.DirEntry{Err: err}: - case <-ctx.Done(): - } - return - } - if len(link.Objects) != 1 { - select { - case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: - case <-ctx.Done(): - } - return + for { + var link lsOutput + if err = dec.Decode(&link); err != nil { + if err != io.EOF { + return err } + return nil + } - if len(link.Objects[0].Links) != 1 { - select { - case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: - case <-ctx.Done(): - } - return - } + if len(link.Objects) != 1 { + return errors.New("unexpected Objects len") + } - l0 := link.Objects[0].Links[0] + if len(link.Objects[0].Links) != 1 { + return errors.New("unexpected Links len") + } - c, err := cid.Decode(l0.Hash) - if err != nil { - select { - case out <- iface.DirEntry{Err: err}: - case <-ctx.Done(): - } - return - } + l0 := link.Objects[0].Links[0] - var ftype iface.FileType - switch l0.Type { - case unixfs.TRaw, unixfs.TFile: - ftype = iface.TFile - case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: - ftype = iface.TDirectory - case unixfs.TSymlink: - ftype = iface.TSymlink - } + c, err := cid.Decode(l0.Hash) + if err != nil { + return err + } - select { - case out <- iface.DirEntry{ - Name: l0.Name, - Cid: c, - Size: l0.Size, - Type: ftype, - Target: l0.Target, - - Mode: l0.Mode, - ModTime: l0.ModTime, - }: - case <-ctx.Done(): - } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink } - }() - return out, nil + select { + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, + + Mode: l0.Mode, + ModTime: l0.ModTime, + }: + case <-ctx.Done(): + return ctx.Err() + } + } } func (api *UnixfsAPI) core() *HttpApi { diff --git a/cmd/ipfs/kubo/pinmfs.go b/cmd/ipfs/kubo/pinmfs.go index c9187145ce3..ec72e326df9 100644 --- a/cmd/ipfs/kubo/pinmfs.go +++ b/cmd/ipfs/kubo/pinmfs.go @@ -183,7 +183,7 @@ func pinMFS(ctx context.Context, node pinMFSNode, cid cid.Cid, svcName string, s // check if MFS pin exists (across all possible states) and inspect its CID pinStatuses := []pinclient.Status{pinclient.StatusQueued, pinclient.StatusPinning, pinclient.StatusPinned, pinclient.StatusFailed} - lsPinCh, lsErrCh := c.Ls(ctx, pinclient.PinOpts.FilterName(pinName), pinclient.PinOpts.FilterStatus(pinStatuses...)) + lsPinCh, lsErrCh := c.GoLs(ctx, pinclient.PinOpts.FilterName(pinName), pinclient.PinOpts.FilterStatus(pinStatuses...)) existingRequestID := "" // is there any pre-existing MFS pin with pinName (for any CID)? pinning := false // is CID for current MFS already being pinned? pinTime := time.Now().UTC() diff --git a/core/commands/ls.go b/core/commands/ls.go index ab914bb0e15..bdd475d96cb 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -1,6 +1,7 @@ package commands import ( + "context" "fmt" "io" "os" @@ -133,23 +134,24 @@ The JSON output contains type information. } } + lsCtx, cancel := context.WithCancel(req.Context) + defer cancel() + for i, fpath := range paths { pth, err := cmdutils.PathOrCidPath(fpath) if err != nil { return err } - results, err := api.Unixfs().Ls(req.Context, pth, - options.Unixfs.ResolveChildren(resolveSize || resolveType)) - if err != nil { - return err - } + results := make(chan iface.DirEntry) + lsErr := make(chan error, 1) + go func() { + lsErr <- api.Unixfs().Ls(lsCtx, pth, results, + options.Unixfs.ResolveChildren(resolveSize || resolveType)) + }() processLink, dirDone = processDir() for link := range results { - if link.Err != nil { - return link.Err - } var ftype unixfs_pb.Data_DataType switch link.Type { case iface.TFile: @@ -170,10 +172,13 @@ The JSON output contains type information. Mode: link.Mode, ModTime: link.ModTime, } - if err := processLink(paths[i], lsLink); err != nil { + if err = processLink(paths[i], lsLink); err != nil { return err } } + if err = <-lsErr; err != nil { + return err + } dirDone(i) } return done() diff --git a/core/commands/pin/pin.go b/core/commands/pin/pin.go index b87760aaff4..428a75b695d 100644 --- a/core/commands/pin/pin.go +++ b/core/commands/pin/pin.go @@ -557,15 +557,16 @@ func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api panic("unhandled pin type") } - pins, err := api.Pin().Ls(req.Context, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) - if err != nil { - return err - } + pins := make(chan coreiface.Pin) + lsErr := make(chan error, 1) + lsCtx, cancel := context.WithCancel(req.Context) + defer cancel() + + go func() { + lsErr <- api.Pin().Ls(lsCtx, pins, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) + }() for p := range pins { - if err := p.Err(); err != nil { - return err - } err = emit(PinLsOutputWrapper{ PinLsObject: PinLsObject{ Type: p.Type(), @@ -577,8 +578,7 @@ func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api return err } } - - return nil + return <-lsErr } const ( diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 3721913e77c..8f6dafc2203 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -285,26 +285,26 @@ Pass '--status=queued,pinning,pinned,failed' to list pins in all states. cmds.DelimitedStringsOption(",", pinStatusOptionName, "Return pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(req.Context) - defer cancel() - c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } - psCh, errCh, err := lsRemote(ctx, req, c) - if err != nil { - return err - } + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + psCh := make(chan pinclient.PinStatusGetter) + lsErr := make(chan error, 1) + go func() { + lsErr <- lsRemote(ctx, req, c, psCh) + }() for ps := range psCh { if err := res.Emit(toRemotePinOutput(ps)); err != nil { return err } } - return <-errCh + return <-lsErr }, Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ @@ -317,7 +317,7 @@ Pass '--status=queued,pinning,pinned,failed' to list pins in all states. } // Executes GET /pins/?query-with-filters -func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client, out chan<- pinclient.PinStatusGetter) error { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName]; nameFound { nameStr := name.(string) @@ -330,7 +330,8 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan for _, rawCID := range cidsRawArr { parsedCID, err := cid.Decode(rawCID) if err != nil { - return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) + close(out) + return fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) } parsedCIDs = append(parsedCIDs, parsedCID) } @@ -342,16 +343,15 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan for _, rawStatus := range statusRawArr { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { - return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) + close(out) + return fmt.Errorf("status %q is not valid", rawStatus) } parsedStatuses = append(parsedStatuses, s) } opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) } - psCh, errCh := c.Ls(ctx, opts...) - - return psCh, errCh, nil + return c.Ls(ctx, out, opts...) } var rmRemotePinCmd = &cmds.Command{ @@ -393,36 +393,37 @@ To list and then remove all pending pin requests, pass an explicit status list: cmds.BoolOption(pinForceOptionName, "Allow removal of multiple pins matching the query without additional confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(req.Context) - defer cancel() - c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } rmIDs := []string{} - if len(req.Arguments) == 0 { - psCh, errCh, err := lsRemote(ctx, req, c) - if err != nil { - return err - } - for ps := range psCh { - rmIDs = append(rmIDs, ps.GetRequestId()) - } - if err = <-errCh; err != nil { - return fmt.Errorf("error while listing remote pins: %v", err) - } - - if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { - return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") - } - } else { + if len(req.Arguments) != 0 { return fmt.Errorf("unexpected argument %q", req.Arguments[0]) } + psCh := make(chan pinclient.PinStatusGetter) + errCh := make(chan error, 1) + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + go func() { + errCh <- lsRemote(ctx, req, c, psCh) + }() + for ps := range psCh { + rmIDs = append(rmIDs, ps.GetRequestId()) + } + if err = <-errCh; err != nil { + return fmt.Errorf("error while listing remote pins: %v", err) + } + + if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { + return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") + } + for _, rmID := range rmIDs { - if err := c.DeleteByID(ctx, rmID); err != nil { + if err = c.DeleteByID(ctx, rmID); err != nil { return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) } } diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 64c65b651c2..878b4c28d0a 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -51,13 +51,14 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp return api.pinning.Flush(ctx) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan coreiface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, pins chan<- coreiface.Pin, opts ...caopts.PinLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Ls") defer span.End() settings, err := caopts.PinLsOptions(opts...) if err != nil { - return nil, err + close(pins) + return err } span.SetAttributes(attribute.String("type", settings.Type)) @@ -65,10 +66,11 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan c switch settings.Type { case "all", "direct", "indirect", "recursive": default: - return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) + close(pins) + return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } - return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name), nil + return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name, pins) } func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { @@ -230,6 +232,7 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro } out := make(chan coreiface.PinStatus) + go func() { defer close(out) for p := range api.pinning.RecursiveKeys(ctx, false) { @@ -254,7 +257,6 @@ type pinInfo struct { pinType string path path.ImmutablePath name string - err error } func (p *pinInfo) Path() path.ImmutablePath { @@ -269,17 +271,12 @@ func (p *pinInfo) Name() string { return p.name } -func (p *pinInfo) Err() error { - return p.err -} - // pinLsAll is an internal function for returning a list of pins // // The caller must keep reading results until the channel is closed to prevent // leaking the goroutine that is fetching pins. -func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string) <-chan coreiface.Pin { - out := make(chan coreiface.Pin, 1) - +func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string, out chan<- coreiface.Pin) error { + defer close(out) emittedSet := cid.NewSet() AddToResultKeys := func(c cid.Cid, pinName, typeStr string) error { @@ -297,87 +294,79 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, return nil } - go func() { - defer close(out) - - var rkeys []cid.Cid - var err error - if typeStr == "recursive" || typeStr == "all" { - for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "recursive"); err != nil { - out <- &pinInfo{err: err} - return - } - rkeys = append(rkeys, streamedCid.Pin.Key) + var rkeys []cid.Cid + var err error + if typeStr == "recursive" || typeStr == "all" { + for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "recursive"); err != nil { + return err + } + rkeys = append(rkeys, streamedCid.Pin.Key) } - if typeStr == "direct" || typeStr == "all" { - for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "direct"); err != nil { - out <- &pinInfo{err: err} - return - } + } + if typeStr == "direct" || typeStr == "all" { + for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err + } + if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "direct"); err != nil { + return err } } - if typeStr == "indirect" { - // We need to first visit the direct pins that have priority - // without emitting them - - for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - emittedSet.Add(streamedCid.Pin.Key) + } + if typeStr == "indirect" { + // We need to first visit the direct pins that have priority + // without emitting them + + for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + emittedSet.Add(streamedCid.Pin.Key) + } - for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - emittedSet.Add(streamedCid.Pin.Key) - rkeys = append(rkeys, streamedCid.Pin.Key) + for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + emittedSet.Add(streamedCid.Pin.Key) + rkeys = append(rkeys, streamedCid.Pin.Key) } - if typeStr == "indirect" || typeStr == "all" { - walkingSet := cid.NewSet() - for _, k := range rkeys { - err = merkledag.Walk( - ctx, merkledag.GetLinksWithDAG(api.dag), k, - func(c cid.Cid) bool { - if !walkingSet.Visit(c) { - return false - } - if emittedSet.Has(c) { - return true // skipped - } - err := AddToResultKeys(c, "", "indirect") - if err != nil { - out <- &pinInfo{err: err} - return false - } - return true - }, - merkledag.SkipRoot(), merkledag.Concurrent(), - ) - if err != nil { - out <- &pinInfo{err: err} - return - } + } + if typeStr == "indirect" || typeStr == "all" { + if len(rkeys) == 0 { + return nil + } + var addErr error + walkingSet := cid.NewSet() + for _, k := range rkeys { + err = merkledag.Walk( + ctx, merkledag.GetLinksWithDAG(api.dag), k, + func(c cid.Cid) bool { + if !walkingSet.Visit(c) { + return false + } + if emittedSet.Has(c) { + return true // skipped + } + addErr = AddToResultKeys(c, "", "indirect") + return addErr == nil + }, + merkledag.SkipRoot(), merkledag.Concurrent(), + ) + if err != nil { + return err + } + if addErr != nil { + return addErr } } - }() + } - return out + return nil } func (api *PinAPI) core() coreiface.CoreAPI { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index dbeeefda426..9d91f09b6ef 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -2,6 +2,7 @@ package coreapi import ( "context" + "errors" "fmt" blockservice "github.com/ipfs/boxo/blockservice" @@ -197,13 +198,15 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) // Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format: // ` ` -func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.UnixfsLsOption) (<-chan coreiface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- coreiface.DirEntry, opts ...options.UnixfsLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "Ls", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() + defer close(out) + settings, err := options.UnixfsLsOptions(opts...) if err != nil { - return nil, err + return err } span.SetAttributes(attribute.Bool("resolvechildren", settings.ResolveChildren)) @@ -213,21 +216,21 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.Unixf dagnode, err := ses.ResolveNode(ctx, p) if err != nil { - return nil, err + return err } dir, err := uio.NewDirectoryFromNode(ses.dag, dagnode) - if err == uio.ErrNotADir { - return uses.lsFromLinks(ctx, dagnode.Links(), settings) - } if err != nil { - return nil, err + if errors.Is(err, uio.ErrNotADir) { + return uses.lsFromLinks(ctx, dagnode.Links(), settings, out) + } + return err } - return uses.lsFromLinksAsync(ctx, dir, settings) + return uses.lsFromDirLinks(ctx, dir, settings, out) } -func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.DirEntry { +func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) (coreiface.DirEntry, error) { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "ProcessLink") defer span.End() if linkres.Link != nil { @@ -235,7 +238,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se } if linkres.Err != nil { - return coreiface.DirEntry{Err: linkres.Err} + return coreiface.DirEntry{}, linkres.Err } lnk := coreiface.DirEntry{ @@ -252,15 +255,13 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se if settings.ResolveChildren { linkNode, err := linkres.Link.GetNode(ctx, api.dag) if err != nil { - lnk.Err = err - break + return coreiface.DirEntry{}, err } if pn, ok := linkNode.(*merkledag.ProtoNode); ok { d, err := ft.FSNodeFromBytes(pn.Data()) if err != nil { - lnk.Err = err - break + return coreiface.DirEntry{}, err } switch d.Type() { case ft.TFile, ft.TRaw: @@ -284,35 +285,50 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se } } - return lnk + return lnk, nil } -func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) { - out := make(chan coreiface.DirEntry, uio.DefaultShardWidth) +func (api *UnixfsAPI) lsFromDirLinks(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { + for l := range dir.EnumLinksAsync(ctx) { + dirEnt, err := api.processLink(ctx, l, settings) // TODO: perf: processing can be done in background and in parallel + if err != nil { + return err + } + select { + case out <- dirEnt: + case <-ctx.Done(): + return nil + } + } + return nil +} +func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { + // Create links channel large enough to not block when writing to out is slower. + links := make(chan coreiface.DirEntry, len(ndlinks)) + errs := make(chan error, 1) go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { + defer close(links) + defer close(errs) + for _, l := range ndlinks { + lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} + lnk, err := api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async + if err != nil { + errs <- err + return + } select { - case out <- api.processLink(ctx, l, settings): // TODO: perf: processing can be done in background and in parallel + case links <- lnk: case <-ctx.Done(): return } } }() - return out, nil -} - -func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) { - links := make(chan coreiface.DirEntry, len(ndlinks)) - for _, l := range ndlinks { - lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} - - links <- api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async + for lnk := range links { + out <- lnk } - close(links) - return links, nil + return <-errs } func (api *UnixfsAPI) core() *CoreAPI { diff --git a/core/coreiface/pin.go b/core/coreiface/pin.go index ed837fc9ce2..e0fd2fb90ed 100644 --- a/core/coreiface/pin.go +++ b/core/coreiface/pin.go @@ -18,9 +18,6 @@ type Pin interface { // Type of the pin Type() string - - // if not nil, an error happened. Everything else should be ignored. - Err() error } // PinStatus holds information about pin health @@ -50,8 +47,9 @@ type PinAPI interface { // tree Add(context.Context, path.Path, ...options.PinAddOption) error - // Ls returns list of pinned objects on this node - Ls(context.Context, ...options.PinLsOption) (<-chan Pin, error) + // Ls returns this node's pinned objects on the provided channel. The + // channel is closed when there are no more pins and an error is returned. + Ls(context.Context, chan<- Pin, ...options.PinLsOption) error // IsPinned returns whether or not the given cid is pinned // and an explanation of why its pinned diff --git a/core/coreiface/tests/block.go b/core/coreiface/tests/block.go index 3b4ca0bc05d..2b5a68a63b8 100644 --- a/core/coreiface/tests/block.go +++ b/core/coreiface/tests/block.go @@ -323,9 +323,17 @@ func (tp *TestSuite) TestBlockPin(t *testing.T) { t.Fatal(err) } - if pins, err := api.Pin().Ls(ctx); err != nil || len(pins) != 0 { + pinCh := make(chan coreiface.Pin) + go func() { + err = api.Pin().Ls(ctx, pinCh) + }() + + for range pinCh { t.Fatal("expected 0 pins") } + if err != nil { + t.Fatal(err) + } res, err := api.Block().Put( ctx, @@ -337,7 +345,7 @@ func (tp *TestSuite) TestBlockPin(t *testing.T) { t.Fatal(err) } - pins, err := accPins(api.Pin().Ls(ctx)) + pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } diff --git a/core/coreiface/tests/pin.go b/core/coreiface/tests/pin.go index fdd7c15ccbf..4c606323fec 100644 --- a/core/coreiface/tests/pin.go +++ b/core/coreiface/tests/pin.go @@ -67,7 +67,7 @@ func (tp *TestSuite) TestPinSimple(t *testing.T) { t.Fatal(err) } - list, err := accPins(api.Pin().Ls(ctx)) + list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func (tp *TestSuite) TestPinSimple(t *testing.T) { t.Fatal(err) } - list, err = accPins(api.Pin().Ls(ctx)) + list, err = accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -143,7 +143,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Fatal(err) } - list, err := accPins(api.Pin().Ls(ctx)) + list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -152,7 +152,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd3.Cid()).String()) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + list, err = accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } @@ -178,7 +178,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd2.Cid()).String()) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } @@ -436,21 +436,21 @@ func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recusive, direct, indirect []cidContainer) { assertPinLsAllConsistency(t, ctx, api) - list, err := accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + list, err := accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } assertPinCids(t, list, recusive...) - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } assertPinCids(t, list, direct...) - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } @@ -500,7 +500,7 @@ func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { // assertPinLsAllConsistency verifies that listing all pins gives the same result as listing the pin types individually func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.CoreAPI) { t.Helper() - allPins, err := accPins(api.Pin().Ls(ctx)) + allPins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -531,7 +531,7 @@ func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.Core } for typeStr, pinProps := range typeMap { - pins, err := accPins(api.Pin().Ls(ctx, pinProps.PinLsOption)) + pins, err := accPins(ctx, api, pinProps.PinLsOption) if err != nil { t.Fatal(err) } @@ -593,19 +593,19 @@ func assertNotPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p pat } } -func accPins(pins <-chan iface.Pin, err error) ([]iface.Pin, error) { - if err != nil { - return nil, err - } - - var result []iface.Pin +func accPins(ctx context.Context, api iface.CoreAPI, opts ...opt.PinLsOption) ([]iface.Pin, error) { + var err error + pins := make(chan iface.Pin) + go func() { + err = api.Pin().Ls(ctx, pins, opts...) + }() + var results []iface.Pin for pin := range pins { - if pin.Err() != nil { - return nil, pin.Err() - } - result = append(result, pin) + results = append(results, pin) } - - return result, nil + if err != nil { + return nil, err + } + return results, nil } diff --git a/core/coreiface/tests/unixfs.go b/core/coreiface/tests/unixfs.go index 9d3362b9aff..987d39b2620 100644 --- a/core/coreiface/tests/unixfs.go +++ b/core/coreiface/tests/unixfs.go @@ -544,7 +544,7 @@ func (tp *TestSuite) TestAddPinned(t *testing.T) { t.Fatal(err) } - pins, err := accPins(api.Pin().Ls(ctx)) + pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -681,14 +681,15 @@ func (tp *TestSuite) TestLs(t *testing.T) { t.Fatal(err) } - entries, err := api.Unixfs().Ls(ctx, p) - if err != nil { - t.Fatal(err) - } + errCh := make(chan error, 1) + entries := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, p, entries) + }() - entry := <-entries - if entry.Err != nil { - t.Fatal(entry.Err) + entry, ok := <-entries + if !ok { + t.Fatal("expected another entry") } if entry.Size != 15 { t.Errorf("expected size = 15, got %d", entry.Size) @@ -702,9 +703,9 @@ func (tp *TestSuite) TestLs(t *testing.T) { if entry.Cid.String() != "QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr" { t.Errorf("expected cid = QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr, got %s", entry.Cid) } - entry = <-entries - if entry.Err != nil { - t.Fatal(entry.Err) + entry, ok = <-entries + if !ok { + t.Fatal("expected another entry") } if entry.Type != coreiface.TSymlink { t.Errorf("wrong type %s", entry.Type) @@ -716,11 +717,12 @@ func (tp *TestSuite) TestLs(t *testing.T) { t.Errorf("expected symlink target to be /foo/bar, got %s", entry.Target) } - if l, ok := <-entries; ok { - t.Errorf("didn't expect a second link") - if l.Err != nil { - t.Error(l.Err) - } + _, ok = <-entries + if ok { + t.Errorf("didn't expect a another link") + } + if err = <-errCh; err != nil { + t.Error(err) } } @@ -779,13 +781,22 @@ func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, p) - if err != nil { + errCh := make(chan error, 1) + links := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, p, links) + }() + + var count int + for range links { + count++ + } + if err = <-errCh; err != nil { t.Fatal(err) } - if len(links) != 0 { - t.Fatalf("expected 0 links, got %d", len(links)) + if count != 0 { + t.Fatalf("expected 0 links, got %d", count) } } @@ -808,13 +819,22 @@ func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, path.FromCid(nd.Cid())) - if err != nil { + errCh := make(chan error, 1) + links := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, path.FromCid(nd.Cid()), links) + }() + + var count int + for range links { + count++ + } + if err = <-errCh; err != nil { t.Fatal(err) } - if len(links) != 0 { - t.Fatalf("expected 0 links, got %d", len(links)) + if count != 0 { + t.Fatalf("expected 0 links, got %d", count) } } diff --git a/core/coreiface/unixfs.go b/core/coreiface/unixfs.go index c0150bd12c6..10371998c20 100644 --- a/core/coreiface/unixfs.go +++ b/core/coreiface/unixfs.go @@ -2,6 +2,7 @@ package iface import ( "context" + "iter" "os" "time" @@ -63,8 +64,6 @@ type DirEntry struct { Mode os.FileMode ModTime time.Time - - Err error } // UnixfsAPI is the basic interface to immutable files in IPFS @@ -81,7 +80,56 @@ type UnixfsAPI interface { // to operations performed on the returned file Get(context.Context, path.Path) (files.Node, error) - // Ls returns the list of links in a directory. Links aren't guaranteed to be - // returned in order - Ls(context.Context, path.Path, ...options.UnixfsLsOption) (<-chan DirEntry, error) + // Ls writes the links in a directory to the DirEntry channel. Links aren't + // guaranteed to be returned in order. If an error occurs or the context is + // canceled, the DirEntry channel is closed and an error is returned. + // + // Example: + // + // dirs := make(chan DirEntry) + // lsErr := make(chan error, 1) + // go func() { + // lsErr <- Ls(ctx, p, dirs) + // }() + // for dirEnt := range dirs { + // fmt.Println("Dir name:", dirEnt.Name) + // } + // err := <-lsErr + // if err != nil { + // return fmt.Errorf("error listing directory: %w", err) + // } + Ls(context.Context, path.Path, chan<- DirEntry, ...options.UnixfsLsOption) error +} + +// LsIter returns a go iterator that allows ranging over DirEntry results. +// Iteration stops if the context is canceled or if the iterator yields an +// error. +// +// Exmaple: +// +// for dirEnt, err := LsIter(ctx, ufsAPI, p) { +// if err != nil { +// return fmt.Errorf("error listing directory: %w", err) +// } +// fmt.Println("Dir name:", dirEnt.Name) +// } +func LsIter(ctx context.Context, api UnixfsAPI, p path.Path, opts ...options.UnixfsLsOption) iter.Seq2[DirEntry, error] { + return func(yield func(DirEntry, error) bool) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() // cancel Ls if done iterating early + + dirs := make(chan DirEntry) + lsErr := make(chan error, 1) + go func() { + lsErr <- api.Ls(ctx, p, dirs, opts...) + }() + for dirEnt := range dirs { + if !yield(dirEnt, nil) { + return + } + } + if err := <-lsErr; err != nil { + yield(DirEntry{}, err) + } + } } diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 3c025040ccf..8cfb0a72752 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.23 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 + github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.37.2 github.com/multiformats/go-multiaddr v0.13.0 @@ -52,6 +52,8 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/gammazero/chanqueue v1.0.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 110c9bf8e17..5ee33c29cdb 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -164,6 +164,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -298,8 +302,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 h1:kiS5+H+6aJeNWWDynuYu/ijgzkBTrInl++VFcNDgq+g= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/go.mod b/go.mod index a9bca9d7f33..b0051da870c 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-version v1.7.0 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c - github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 + github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 @@ -125,6 +125,8 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/gammazero/chanqueue v1.0.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-kit/log v0.2.1 // indirect diff --git a/go.sum b/go.sum index c29dbd816bf..9054a1f7033 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -362,8 +366,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 h1:kiS5+H+6aJeNWWDynuYu/ijgzkBTrInl++VFcNDgq+g= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 93ab3f923e0..05847e9e4f5 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -119,7 +119,7 @@ require ( github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 // indirect + github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 11be0c149fd..e0f4abd3c91 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -162,6 +162,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghostiam/protogetter v0.3.6 h1:R7qEWaSgFCsy20yYHNIJsU9ZOb8TziSRRxuAOTVKeOk= github.com/ghostiam/protogetter v0.3.6/go.mod h1:7lpeDnEJ1ZjL/YtyoN99ljO4z0pd3H0d18/t2dPBxHw= @@ -318,8 +322,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw= -github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 h1:kiS5+H+6aJeNWWDynuYu/ijgzkBTrInl++VFcNDgq+g= +github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= From 433444b6097685b26843e517fb934782d56be99a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 3 Dec 2024 21:49:18 +0100 Subject: [PATCH 16/16] fix(cmds/add): disallow --wrap with --to-files (#10612) Close #10611 Co-authored-by: Marcin Rataj --- core/commands/add.go | 9 +++++++++ docs/changelogs/v0.33.md | 7 +++++++ test/sharness/t0040-add-and-cat.sh | 13 +++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 90861302551..73491c03b44 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -288,6 +288,10 @@ See 'dag export' and 'dag import' for more information. return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) } + if wrap && toFilesSet { + return fmt.Errorf("%s and %s options are not compatible", wrapOptionName, toFilesOptionName) + } + hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr)) @@ -373,6 +377,11 @@ See 'dag export' and 'dag import' for more information. // creating MFS pointers when optional --to-files is set if toFilesSet { + if addit.Name() == "" { + errCh <- fmt.Errorf("%s: cannot add unnamed files to MFS", toFilesOptionName) + return + } + if toFilesStr == "" { toFilesStr = "/" } diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index a5d8ef2ef34..1adf16dd0bb 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -6,6 +6,9 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo) + - [Using default `libp2p_rcmgr` metrics](#using-default-libp2p_rcmgr--metrics) + - [`ipfs add --to-files` no longer works with `--wrap`](#ipfs-add---to-files-no-longer-works-with---wrap) - [๐Ÿ“ฆ๏ธ Dependency updates](#-dependency-updates) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -24,6 +27,10 @@ Bespoke rcmgr metrics [were removed](https://github.com/ipfs/kubo/pull/9947), Ku This makes it easier to compare Kubo with custom implementations based on go-libp2p. If you depended on removed ones, please fill an issue to add them to the upstream [go-libp2p](https://github.com/libp2p/go-libp2p). +#### `ipfs add --to-files` no longer works with `--wrap` + +Onboarding files and directories with `ipfs add --to-files` now requires non-empty names. due to this, The `--to-files` and `--wrap` options are now mutually exclusive ([#10612](https://github.com/ipfs/kubo/issues/10612)). + #### ๐Ÿ“ฆ๏ธ Dependency updates - update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 142ab8ec18c..c7232bedd8c 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -355,10 +355,10 @@ test_add_cat_file() { test_cmp expected actual ' - test_must_fail "ipfs add with multiple files of same name but different dirs fails" ' + test_expect_success "ipfs add with multiple files of same name but different dirs fails" ' mkdir -p mountdir/same-file/ && cp mountdir/hello.txt mountdir/same-file/hello.txt && - ipfs add mountdir/hello.txt mountdir/same-file/hello.txt >actual && + test_expect_code 1 ipfs add mountdir/hello.txt mountdir/same-file/hello.txt >actual && rm mountdir/same-file/hello.txt && rmdir mountdir/same-file ' @@ -469,6 +469,15 @@ test_add_cat_file() { ipfs files rm -r --force /mfs ' + # confirm -w and --to-files are exclusive + # context: https://github.com/ipfs/kubo/issues/10611 + test_expect_success "ipfs add -r -w dir --to-files /mfs/subdir5/ errors (-w and --to-files are exclusive)" ' + ipfs files mkdir -p /mfs/subdir5 && + test_expect_code 1 ipfs add -r -w test --to-files /mfs/subdir5/ >actual 2>&1 && + test_should_contain "Error" actual && + ipfs files rm -r --force /mfs + ' + } test_add_cat_5MB() {