diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index c6cac2c78de..c3846b03fab 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -45,7 +45,7 @@ jobs: make -j "$PARALLEL" test/unit/gotest.junit.xml && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]] - name: Upload coverage to Codecov - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7 if: failure() || success() with: name: unittests diff --git a/.github/workflows/sharness.yml b/.github/workflows/sharness.yml index ac91b132161..354cad1029e 100644 --- a/.github/workflows/sharness.yml +++ b/.github/workflows/sharness.yml @@ -55,7 +55,7 @@ jobs: # increasing parallelism beyond 10 doesn't speed up the tests much PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }} - name: Upload coverage report - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7 if: failure() || success() with: name: sharness diff --git a/client/rpc/pin.go b/client/rpc/pin.go index 6e8e942ac0e..2b1ef825801 100644 --- a/client/rpc/pin.go +++ b/client/rpc/pin.go @@ -62,10 +62,12 @@ type pinLsObject struct { Type string } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan iface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, pins chan<- iface.Pin, opts ...caopts.PinLsOption) error { + defer close(pins) + options, err := caopts.PinLsOptions(opts...) if err != nil { - return nil, err + return err } res, err := api.core().Request("pin/ls"). @@ -73,48 +75,32 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan i Option("stream", true). Send(ctx) if err != nil { - return nil, err + return err } - - pins := make(chan iface.Pin) - go func(ch chan<- iface.Pin) { - defer res.Output.Close() - defer close(ch) - - dec := json.NewDecoder(res.Output) - var out pinLsObject - for { - switch err := dec.Decode(&out); err { - case nil: - case io.EOF: - return - default: - select { - case ch <- pin{err: err}: - return - case <-ctx.Done(): - return - } + defer res.Output.Close() + + dec := json.NewDecoder(res.Output) + var out pinLsObject + for { + err := dec.Decode(&out) + if err != nil { + if err != io.EOF { + return err } + return nil + } - c, err := cid.Parse(out.Cid) - if err != nil { - select { - case ch <- pin{err: err}: - return - case <-ctx.Done(): - return - } - } + c, err := cid.Parse(out.Cid) + if err != nil { + return err + } - select { - case ch <- pin{typ: out.Type, name: out.Name, path: path.FromCid(c)}: - case <-ctx.Done(): - return - } + select { + case pins <- pin{typ: out.Type, name: out.Name, path: path.FromCid(c)}: + case <-ctx.Done(): + return ctx.Err() } - }(pins) - return pins, nil + } } // IsPinned returns whether or not the given cid is pinned diff --git a/client/rpc/unixfs.go b/client/rpc/unixfs.go index 3ba2c1c15f0..316cc21a8ec 100644 --- a/client/rpc/unixfs.go +++ b/client/rpc/unixfs.go @@ -144,10 +144,12 @@ type lsOutput struct { Objects []lsObject } -func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.UnixfsLsOption) (<-chan iface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- iface.DirEntry, opts ...caopts.UnixfsLsOption) error { + defer close(out) + options, err := caopts.UnixfsLsOptions(opts...) if err != nil { - return nil, err + return err } resp, err := api.core().Request("ls", p.String()). @@ -156,86 +158,64 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...caopts.Unixfs Option("stream", true). Send(ctx) if err != nil { - return nil, err + return err } if resp.Error != nil { - return nil, resp.Error + return err } + defer resp.Close() dec := json.NewDecoder(resp.Output) - out := make(chan iface.DirEntry) - - go func() { - defer resp.Close() - defer close(out) - - for { - var link lsOutput - if err := dec.Decode(&link); err != nil { - if err == io.EOF { - return - } - select { - case out <- iface.DirEntry{Err: err}: - case <-ctx.Done(): - } - return - } - if len(link.Objects) != 1 { - select { - case out <- iface.DirEntry{Err: errors.New("unexpected Objects len")}: - case <-ctx.Done(): - } - return + for { + var link lsOutput + if err = dec.Decode(&link); err != nil { + if err != io.EOF { + return err } + return nil + } - if len(link.Objects[0].Links) != 1 { - select { - case out <- iface.DirEntry{Err: errors.New("unexpected Links len")}: - case <-ctx.Done(): - } - return - } + if len(link.Objects) != 1 { + return errors.New("unexpected Objects len") + } - l0 := link.Objects[0].Links[0] + if len(link.Objects[0].Links) != 1 { + return errors.New("unexpected Links len") + } - c, err := cid.Decode(l0.Hash) - if err != nil { - select { - case out <- iface.DirEntry{Err: err}: - case <-ctx.Done(): - } - return - } + l0 := link.Objects[0].Links[0] - var ftype iface.FileType - switch l0.Type { - case unixfs.TRaw, unixfs.TFile: - ftype = iface.TFile - case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: - ftype = iface.TDirectory - case unixfs.TSymlink: - ftype = iface.TSymlink - } + c, err := cid.Decode(l0.Hash) + if err != nil { + return err + } - select { - case out <- iface.DirEntry{ - Name: l0.Name, - Cid: c, - Size: l0.Size, - Type: ftype, - Target: l0.Target, - - Mode: l0.Mode, - ModTime: l0.ModTime, - }: - case <-ctx.Done(): - } + var ftype iface.FileType + switch l0.Type { + case unixfs.TRaw, unixfs.TFile: + ftype = iface.TFile + case unixfs.THAMTShard, unixfs.TDirectory, unixfs.TMetadata: + ftype = iface.TDirectory + case unixfs.TSymlink: + ftype = iface.TSymlink } - }() - return out, nil + select { + case out <- iface.DirEntry{ + Name: l0.Name, + Cid: c, + Size: l0.Size, + Type: ftype, + Target: l0.Target, + + Mode: l0.Mode, + ModTime: l0.ModTime, + }: + case <-ctx.Done(): + return ctx.Err() + } + } } func (api *UnixfsAPI) core() *HttpApi { diff --git a/cmd/ipfs/kubo/pinmfs.go b/cmd/ipfs/kubo/pinmfs.go index c9187145ce3..ec72e326df9 100644 --- a/cmd/ipfs/kubo/pinmfs.go +++ b/cmd/ipfs/kubo/pinmfs.go @@ -183,7 +183,7 @@ func pinMFS(ctx context.Context, node pinMFSNode, cid cid.Cid, svcName string, s // check if MFS pin exists (across all possible states) and inspect its CID pinStatuses := []pinclient.Status{pinclient.StatusQueued, pinclient.StatusPinning, pinclient.StatusPinned, pinclient.StatusFailed} - lsPinCh, lsErrCh := c.Ls(ctx, pinclient.PinOpts.FilterName(pinName), pinclient.PinOpts.FilterStatus(pinStatuses...)) + lsPinCh, lsErrCh := c.GoLs(ctx, pinclient.PinOpts.FilterName(pinName), pinclient.PinOpts.FilterStatus(pinStatuses...)) existingRequestID := "" // is there any pre-existing MFS pin with pinName (for any CID)? pinning := false // is CID for current MFS already being pinned? pinTime := time.Now().UTC() diff --git a/config/bootstrap_peers.go b/config/bootstrap_peers.go index 1671d9f8156..91f8ea1afa4 100644 --- a/config/bootstrap_peers.go +++ b/config/bootstrap_peers.go @@ -16,11 +16,12 @@ import ( // import dependency issue. TODO: move this into a config/default/ package. var DefaultBootstrapAddresses = []string{ "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", - "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", // rust-libp2p-server "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", - "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io - "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io + "/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8", // js-libp2p-amino-dht-bootstrapper + "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io + "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io } // ErrInvalidPeerAddr signals an address is not a valid peer address. diff --git a/core/commands/add.go b/core/commands/add.go index 90861302551..73491c03b44 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -288,6 +288,10 @@ See 'dag export' and 'dag import' for more information. return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName) } + if wrap && toFilesSet { + return fmt.Errorf("%s and %s options are not compatible", wrapOptionName, toFilesOptionName) + } + hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)] if !ok { return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr)) @@ -373,6 +377,11 @@ See 'dag export' and 'dag import' for more information. // creating MFS pointers when optional --to-files is set if toFilesSet { + if addit.Name() == "" { + errCh <- fmt.Errorf("%s: cannot add unnamed files to MFS", toFilesOptionName) + return + } + if toFilesStr == "" { toFilesStr = "/" } diff --git a/core/commands/bitswap.go b/core/commands/bitswap.go index 07f91fb0f52..7f7ab963651 100644 --- a/core/commands/bitswap.go +++ b/core/commands/bitswap.go @@ -5,7 +5,6 @@ import ( "io" cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" - e "github.com/ipfs/kubo/core/commands/e" humanize "github.com/dustin/go-humanize" bitswap "github.com/ipfs/boxo/bitswap" @@ -53,10 +52,7 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`, return ErrNotOnline } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } + bs := nd.Bitswap pstr, found := req.Options[peerOptionName].(string) if found { @@ -112,12 +108,7 @@ var bitswapStatCmd = &cmds.Command{ return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline) } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } - - st, err := bs.Stat() + st, err := nd.Bitswap.Stat() if err != nil { return err } @@ -134,7 +125,6 @@ var bitswapStatCmd = &cmds.Command{ human, _ := req.Options[bitswapHumanOptionName].(bool) fmt.Fprintln(w, "bitswap status") - fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize) fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived) fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent) if human { @@ -190,17 +180,12 @@ prints the ledger associated with a given peer. return ErrNotOnline } - bs, ok := nd.Exchange.(*bitswap.Bitswap) - if !ok { - return e.TypeErr(bs, nd.Exchange) - } - partner, err := peer.Decode(req.Arguments[0]) if err != nil { return err } - return cmds.EmitOnce(res, bs.LedgerForPeer(partner)) + return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner)) }, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error { diff --git a/core/commands/ls.go b/core/commands/ls.go index ab914bb0e15..bdd475d96cb 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -1,6 +1,7 @@ package commands import ( + "context" "fmt" "io" "os" @@ -133,23 +134,24 @@ The JSON output contains type information. } } + lsCtx, cancel := context.WithCancel(req.Context) + defer cancel() + for i, fpath := range paths { pth, err := cmdutils.PathOrCidPath(fpath) if err != nil { return err } - results, err := api.Unixfs().Ls(req.Context, pth, - options.Unixfs.ResolveChildren(resolveSize || resolveType)) - if err != nil { - return err - } + results := make(chan iface.DirEntry) + lsErr := make(chan error, 1) + go func() { + lsErr <- api.Unixfs().Ls(lsCtx, pth, results, + options.Unixfs.ResolveChildren(resolveSize || resolveType)) + }() processLink, dirDone = processDir() for link := range results { - if link.Err != nil { - return link.Err - } var ftype unixfs_pb.Data_DataType switch link.Type { case iface.TFile: @@ -170,10 +172,13 @@ The JSON output contains type information. Mode: link.Mode, ModTime: link.ModTime, } - if err := processLink(paths[i], lsLink); err != nil { + if err = processLink(paths[i], lsLink); err != nil { return err } } + if err = <-lsErr; err != nil { + return err + } dirDone(i) } return done() diff --git a/core/commands/pin/pin.go b/core/commands/pin/pin.go index b87760aaff4..428a75b695d 100644 --- a/core/commands/pin/pin.go +++ b/core/commands/pin/pin.go @@ -557,15 +557,16 @@ func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api panic("unhandled pin type") } - pins, err := api.Pin().Ls(req.Context, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) - if err != nil { - return err - } + pins := make(chan coreiface.Pin) + lsErr := make(chan error, 1) + lsCtx, cancel := context.WithCancel(req.Context) + defer cancel() + + go func() { + lsErr <- api.Pin().Ls(lsCtx, pins, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) + }() for p := range pins { - if err := p.Err(); err != nil { - return err - } err = emit(PinLsOutputWrapper{ PinLsObject: PinLsObject{ Type: p.Type(), @@ -577,8 +578,7 @@ func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api return err } } - - return nil + return <-lsErr } const ( diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 3721913e77c..8f6dafc2203 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -285,26 +285,26 @@ Pass '--status=queued,pinning,pinned,failed' to list pins in all states. cmds.DelimitedStringsOption(",", pinStatusOptionName, "Return pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(req.Context) - defer cancel() - c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } - psCh, errCh, err := lsRemote(ctx, req, c) - if err != nil { - return err - } + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + psCh := make(chan pinclient.PinStatusGetter) + lsErr := make(chan error, 1) + go func() { + lsErr <- lsRemote(ctx, req, c, psCh) + }() for ps := range psCh { if err := res.Emit(toRemotePinOutput(ps)); err != nil { return err } } - return <-errCh + return <-lsErr }, Type: RemotePinOutput{}, Encoders: cmds.EncoderMap{ @@ -317,7 +317,7 @@ Pass '--status=queued,pinning,pinned,failed' to list pins in all states. } // Executes GET /pins/?query-with-filters -func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan pinclient.PinStatusGetter, chan error, error) { +func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client, out chan<- pinclient.PinStatusGetter) error { opts := []pinclient.LsOption{} if name, nameFound := req.Options[pinNameOptionName]; nameFound { nameStr := name.(string) @@ -330,7 +330,8 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan for _, rawCID := range cidsRawArr { parsedCID, err := cid.Decode(rawCID) if err != nil { - return nil, nil, fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) + close(out) + return fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) } parsedCIDs = append(parsedCIDs, parsedCID) } @@ -342,16 +343,15 @@ func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client) (chan for _, rawStatus := range statusRawArr { s := pinclient.Status(rawStatus) if s.String() == string(pinclient.StatusUnknown) { - return nil, nil, fmt.Errorf("status %q is not valid", rawStatus) + close(out) + return fmt.Errorf("status %q is not valid", rawStatus) } parsedStatuses = append(parsedStatuses, s) } opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) } - psCh, errCh := c.Ls(ctx, opts...) - - return psCh, errCh, nil + return c.Ls(ctx, out, opts...) } var rmRemotePinCmd = &cmds.Command{ @@ -393,36 +393,37 @@ To list and then remove all pending pin requests, pass an explicit status list: cmds.BoolOption(pinForceOptionName, "Allow removal of multiple pins matching the query without additional confirmation.").WithDefault(false), }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { - ctx, cancel := context.WithCancel(req.Context) - defer cancel() - c, err := getRemotePinServiceFromRequest(req, env) if err != nil { return err } rmIDs := []string{} - if len(req.Arguments) == 0 { - psCh, errCh, err := lsRemote(ctx, req, c) - if err != nil { - return err - } - for ps := range psCh { - rmIDs = append(rmIDs, ps.GetRequestId()) - } - if err = <-errCh; err != nil { - return fmt.Errorf("error while listing remote pins: %v", err) - } - - if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { - return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") - } - } else { + if len(req.Arguments) != 0 { return fmt.Errorf("unexpected argument %q", req.Arguments[0]) } + psCh := make(chan pinclient.PinStatusGetter) + errCh := make(chan error, 1) + ctx, cancel := context.WithCancel(req.Context) + defer cancel() + + go func() { + errCh <- lsRemote(ctx, req, c, psCh) + }() + for ps := range psCh { + rmIDs = append(rmIDs, ps.GetRequestId()) + } + if err = <-errCh; err != nil { + return fmt.Errorf("error while listing remote pins: %v", err) + } + + if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { + return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") + } + for _, rmID := range rmIDs { - if err := c.DeleteByID(ctx, rmID); err != nil { + if err = c.DeleteByID(ctx, rmID); err != nil { return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) } } diff --git a/core/core.go b/core/core.go index 0c9333e0683..54c98752762 100644 --- a/core/core.go +++ b/core/core.go @@ -19,6 +19,7 @@ import ( pin "github.com/ipfs/boxo/pinning/pinner" "github.com/ipfs/go-datastore" + bitswap "github.com/ipfs/boxo/bitswap" bserv "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" @@ -102,7 +103,8 @@ type IpfsNode struct { UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks - Exchange exchange.Interface // the block exchange + strategy (bitswap) + Exchange exchange.Interface // the block exchange + strategy + Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance Namesys namesys.NameSystem // the name system, resolves paths to hashes Provider provider.System // the value provider system IpnsRepub *ipnsrp.Republisher `optional:"true"` diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index 22b3aa25c24..878b4c28d0a 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -44,20 +44,21 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp return fmt.Errorf("pin: %s", err) } - if err := api.provider.Provide(dagNode.Cid()); err != nil { + if err := api.provider.Provide(ctx, dagNode.Cid(), true); err != nil { return err } return api.pinning.Flush(ctx) } -func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan coreiface.Pin, error) { +func (api *PinAPI) Ls(ctx context.Context, pins chan<- coreiface.Pin, opts ...caopts.PinLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.PinAPI", "Ls") defer span.End() settings, err := caopts.PinLsOptions(opts...) if err != nil { - return nil, err + close(pins) + return err } span.SetAttributes(attribute.String("type", settings.Type)) @@ -65,10 +66,11 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan c switch settings.Type { case "all", "direct", "indirect", "recursive": default: - return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) + close(pins) + return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) } - return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name), nil + return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name, pins) } func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) { @@ -230,6 +232,7 @@ func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, erro } out := make(chan coreiface.PinStatus) + go func() { defer close(out) for p := range api.pinning.RecursiveKeys(ctx, false) { @@ -254,7 +257,6 @@ type pinInfo struct { pinType string path path.ImmutablePath name string - err error } func (p *pinInfo) Path() path.ImmutablePath { @@ -269,17 +271,12 @@ func (p *pinInfo) Name() string { return p.name } -func (p *pinInfo) Err() error { - return p.err -} - // pinLsAll is an internal function for returning a list of pins // // The caller must keep reading results until the channel is closed to prevent // leaking the goroutine that is fetching pins. -func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string) <-chan coreiface.Pin { - out := make(chan coreiface.Pin, 1) - +func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string, out chan<- coreiface.Pin) error { + defer close(out) emittedSet := cid.NewSet() AddToResultKeys := func(c cid.Cid, pinName, typeStr string) error { @@ -297,87 +294,79 @@ func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, return nil } - go func() { - defer close(out) - - var rkeys []cid.Cid - var err error - if typeStr == "recursive" || typeStr == "all" { - for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "recursive"); err != nil { - out <- &pinInfo{err: err} - return - } - rkeys = append(rkeys, streamedCid.Pin.Key) + var rkeys []cid.Cid + var err error + if typeStr == "recursive" || typeStr == "all" { + for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "recursive"); err != nil { + return err + } + rkeys = append(rkeys, streamedCid.Pin.Key) } - if typeStr == "direct" || typeStr == "all" { - for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "direct"); err != nil { - out <- &pinInfo{err: err} - return - } + } + if typeStr == "direct" || typeStr == "all" { + for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err + } + if err = AddToResultKeys(streamedCid.Pin.Key, streamedCid.Pin.Name, "direct"); err != nil { + return err } } - if typeStr == "indirect" { - // We need to first visit the direct pins that have priority - // without emitting them - - for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - emittedSet.Add(streamedCid.Pin.Key) + } + if typeStr == "indirect" { + // We need to first visit the direct pins that have priority + // without emitting them + + for streamedCid := range api.pinning.DirectKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + emittedSet.Add(streamedCid.Pin.Key) + } - for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { - if streamedCid.Err != nil { - out <- &pinInfo{err: streamedCid.Err} - return - } - emittedSet.Add(streamedCid.Pin.Key) - rkeys = append(rkeys, streamedCid.Pin.Key) + for streamedCid := range api.pinning.RecursiveKeys(ctx, detailed) { + if streamedCid.Err != nil { + return streamedCid.Err } + emittedSet.Add(streamedCid.Pin.Key) + rkeys = append(rkeys, streamedCid.Pin.Key) } - if typeStr == "indirect" || typeStr == "all" { - walkingSet := cid.NewSet() - for _, k := range rkeys { - err = merkledag.Walk( - ctx, merkledag.GetLinksWithDAG(api.dag), k, - func(c cid.Cid) bool { - if !walkingSet.Visit(c) { - return false - } - if emittedSet.Has(c) { - return true // skipped - } - err := AddToResultKeys(c, "", "indirect") - if err != nil { - out <- &pinInfo{err: err} - return false - } - return true - }, - merkledag.SkipRoot(), merkledag.Concurrent(), - ) - if err != nil { - out <- &pinInfo{err: err} - return - } + } + if typeStr == "indirect" || typeStr == "all" { + if len(rkeys) == 0 { + return nil + } + var addErr error + walkingSet := cid.NewSet() + for _, k := range rkeys { + err = merkledag.Walk( + ctx, merkledag.GetLinksWithDAG(api.dag), k, + func(c cid.Cid) bool { + if !walkingSet.Visit(c) { + return false + } + if emittedSet.Has(c) { + return true // skipped + } + addErr = AddToResultKeys(c, "", "indirect") + return addErr == nil + }, + merkledag.SkipRoot(), merkledag.Concurrent(), + ) + if err != nil { + return err + } + if addErr != nil { + return addErr } } - }() + } - return out + return nil } func (api *PinAPI) core() coreiface.CoreAPI { diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index e175488f37f..9d91f09b6ef 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -2,6 +2,7 @@ package coreapi import ( "context" + "errors" "fmt" blockservice "github.com/ipfs/boxo/blockservice" @@ -173,7 +174,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options } if !settings.OnlyHash { - if err := api.provider.Provide(nd.Cid()); err != nil { + if err := api.provider.Provide(ctx, nd.Cid(), true); err != nil { return path.ImmutablePath{}, err } } @@ -197,13 +198,15 @@ func (api *UnixfsAPI) Get(ctx context.Context, p path.Path) (files.Node, error) // Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format: // ` ` -func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.UnixfsLsOption) (<-chan coreiface.DirEntry, error) { +func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, out chan<- coreiface.DirEntry, opts ...options.UnixfsLsOption) error { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "Ls", trace.WithAttributes(attribute.String("path", p.String()))) defer span.End() + defer close(out) + settings, err := options.UnixfsLsOptions(opts...) if err != nil { - return nil, err + return err } span.SetAttributes(attribute.Bool("resolvechildren", settings.ResolveChildren)) @@ -213,21 +216,21 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p path.Path, opts ...options.Unixf dagnode, err := ses.ResolveNode(ctx, p) if err != nil { - return nil, err + return err } dir, err := uio.NewDirectoryFromNode(ses.dag, dagnode) - if err == uio.ErrNotADir { - return uses.lsFromLinks(ctx, dagnode.Links(), settings) - } if err != nil { - return nil, err + if errors.Is(err, uio.ErrNotADir) { + return uses.lsFromLinks(ctx, dagnode.Links(), settings, out) + } + return err } - return uses.lsFromLinksAsync(ctx, dir, settings) + return uses.lsFromDirLinks(ctx, dir, settings, out) } -func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.DirEntry { +func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) (coreiface.DirEntry, error) { ctx, span := tracing.Span(ctx, "CoreAPI.UnixfsAPI", "ProcessLink") defer span.End() if linkres.Link != nil { @@ -235,7 +238,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se } if linkres.Err != nil { - return coreiface.DirEntry{Err: linkres.Err} + return coreiface.DirEntry{}, linkres.Err } lnk := coreiface.DirEntry{ @@ -252,15 +255,13 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se if settings.ResolveChildren { linkNode, err := linkres.Link.GetNode(ctx, api.dag) if err != nil { - lnk.Err = err - break + return coreiface.DirEntry{}, err } if pn, ok := linkNode.(*merkledag.ProtoNode); ok { d, err := ft.FSNodeFromBytes(pn.Data()) if err != nil { - lnk.Err = err - break + return coreiface.DirEntry{}, err } switch d.Type() { case ft.TFile, ft.TRaw: @@ -284,35 +285,50 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se } } - return lnk + return lnk, nil } -func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) { - out := make(chan coreiface.DirEntry, uio.DefaultShardWidth) +func (api *UnixfsAPI) lsFromDirLinks(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { + for l := range dir.EnumLinksAsync(ctx) { + dirEnt, err := api.processLink(ctx, l, settings) // TODO: perf: processing can be done in background and in parallel + if err != nil { + return err + } + select { + case out <- dirEnt: + case <-ctx.Done(): + return nil + } + } + return nil +} +func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings, out chan<- coreiface.DirEntry) error { + // Create links channel large enough to not block when writing to out is slower. + links := make(chan coreiface.DirEntry, len(ndlinks)) + errs := make(chan error, 1) go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { + defer close(links) + defer close(errs) + for _, l := range ndlinks { + lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} + lnk, err := api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async + if err != nil { + errs <- err + return + } select { - case out <- api.processLink(ctx, l, settings): // TODO: perf: processing can be done in background and in parallel + case links <- lnk: case <-ctx.Done(): return } } }() - return out, nil -} - -func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) { - links := make(chan coreiface.DirEntry, len(ndlinks)) - for _, l := range ndlinks { - lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}} - - links <- api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async + for lnk := range links { + out <- lnk } - close(links) - return links, nil + return <-errs } func (api *UnixfsAPI) core() *CoreAPI { diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index 4f00ad71202..f0c4c3c1cff 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // WebUI version confirmed to work with this Kubo version -const WebUIPath = "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm" // v4.4.0 +const WebUIPath = "/ipfs/bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4" // v4.4.1 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeibgic2ex3fvzkinhy6k6aqyv3zy2o7bkbsmrzvzka24xetv7eeadm", "/ipfs/bafybeid4uxz7klxcu3ffsnmn64r7ihvysamlj4ohl5h2orjsffuegcpaeq", "/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy", "/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4", diff --git a/core/coreiface/pin.go b/core/coreiface/pin.go index ed837fc9ce2..e0fd2fb90ed 100644 --- a/core/coreiface/pin.go +++ b/core/coreiface/pin.go @@ -18,9 +18,6 @@ type Pin interface { // Type of the pin Type() string - - // if not nil, an error happened. Everything else should be ignored. - Err() error } // PinStatus holds information about pin health @@ -50,8 +47,9 @@ type PinAPI interface { // tree Add(context.Context, path.Path, ...options.PinAddOption) error - // Ls returns list of pinned objects on this node - Ls(context.Context, ...options.PinLsOption) (<-chan Pin, error) + // Ls returns this node's pinned objects on the provided channel. The + // channel is closed when there are no more pins and an error is returned. + Ls(context.Context, chan<- Pin, ...options.PinLsOption) error // IsPinned returns whether or not the given cid is pinned // and an explanation of why its pinned diff --git a/core/coreiface/tests/block.go b/core/coreiface/tests/block.go index 3b4ca0bc05d..2b5a68a63b8 100644 --- a/core/coreiface/tests/block.go +++ b/core/coreiface/tests/block.go @@ -323,9 +323,17 @@ func (tp *TestSuite) TestBlockPin(t *testing.T) { t.Fatal(err) } - if pins, err := api.Pin().Ls(ctx); err != nil || len(pins) != 0 { + pinCh := make(chan coreiface.Pin) + go func() { + err = api.Pin().Ls(ctx, pinCh) + }() + + for range pinCh { t.Fatal("expected 0 pins") } + if err != nil { + t.Fatal(err) + } res, err := api.Block().Put( ctx, @@ -337,7 +345,7 @@ func (tp *TestSuite) TestBlockPin(t *testing.T) { t.Fatal(err) } - pins, err := accPins(api.Pin().Ls(ctx)) + pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } diff --git a/core/coreiface/tests/pin.go b/core/coreiface/tests/pin.go index fdd7c15ccbf..4c606323fec 100644 --- a/core/coreiface/tests/pin.go +++ b/core/coreiface/tests/pin.go @@ -67,7 +67,7 @@ func (tp *TestSuite) TestPinSimple(t *testing.T) { t.Fatal(err) } - list, err := accPins(api.Pin().Ls(ctx)) + list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func (tp *TestSuite) TestPinSimple(t *testing.T) { t.Fatal(err) } - list, err = accPins(api.Pin().Ls(ctx)) + list, err = accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -143,7 +143,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Fatal(err) } - list, err := accPins(api.Pin().Ls(ctx)) + list, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -152,7 +152,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd3.Cid()).String()) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + list, err = accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } @@ -178,7 +178,7 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd2.Cid()).String()) } - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } @@ -436,21 +436,21 @@ func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recusive, direct, indirect []cidContainer) { assertPinLsAllConsistency(t, ctx, api) - list, err := accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) + list, err := accPins(ctx, api, opt.Pin.Ls.Recursive()) if err != nil { t.Fatal(err) } assertPinCids(t, list, recusive...) - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Direct())) + list, err = accPins(ctx, api, opt.Pin.Ls.Direct()) if err != nil { t.Fatal(err) } assertPinCids(t, list, direct...) - list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) + list, err = accPins(ctx, api, opt.Pin.Ls.Indirect()) if err != nil { t.Fatal(err) } @@ -500,7 +500,7 @@ func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { // assertPinLsAllConsistency verifies that listing all pins gives the same result as listing the pin types individually func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.CoreAPI) { t.Helper() - allPins, err := accPins(api.Pin().Ls(ctx)) + allPins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -531,7 +531,7 @@ func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.Core } for typeStr, pinProps := range typeMap { - pins, err := accPins(api.Pin().Ls(ctx, pinProps.PinLsOption)) + pins, err := accPins(ctx, api, pinProps.PinLsOption) if err != nil { t.Fatal(err) } @@ -593,19 +593,19 @@ func assertNotPinned(t *testing.T, ctx context.Context, api iface.CoreAPI, p pat } } -func accPins(pins <-chan iface.Pin, err error) ([]iface.Pin, error) { - if err != nil { - return nil, err - } - - var result []iface.Pin +func accPins(ctx context.Context, api iface.CoreAPI, opts ...opt.PinLsOption) ([]iface.Pin, error) { + var err error + pins := make(chan iface.Pin) + go func() { + err = api.Pin().Ls(ctx, pins, opts...) + }() + var results []iface.Pin for pin := range pins { - if pin.Err() != nil { - return nil, pin.Err() - } - result = append(result, pin) + results = append(results, pin) } - - return result, nil + if err != nil { + return nil, err + } + return results, nil } diff --git a/core/coreiface/tests/unixfs.go b/core/coreiface/tests/unixfs.go index 9d3362b9aff..987d39b2620 100644 --- a/core/coreiface/tests/unixfs.go +++ b/core/coreiface/tests/unixfs.go @@ -544,7 +544,7 @@ func (tp *TestSuite) TestAddPinned(t *testing.T) { t.Fatal(err) } - pins, err := accPins(api.Pin().Ls(ctx)) + pins, err := accPins(ctx, api) if err != nil { t.Fatal(err) } @@ -681,14 +681,15 @@ func (tp *TestSuite) TestLs(t *testing.T) { t.Fatal(err) } - entries, err := api.Unixfs().Ls(ctx, p) - if err != nil { - t.Fatal(err) - } + errCh := make(chan error, 1) + entries := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, p, entries) + }() - entry := <-entries - if entry.Err != nil { - t.Fatal(entry.Err) + entry, ok := <-entries + if !ok { + t.Fatal("expected another entry") } if entry.Size != 15 { t.Errorf("expected size = 15, got %d", entry.Size) @@ -702,9 +703,9 @@ func (tp *TestSuite) TestLs(t *testing.T) { if entry.Cid.String() != "QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr" { t.Errorf("expected cid = QmX3qQVKxDGz3URVC3861Z3CKtQKGBn6ffXRBBWGMFz9Lr, got %s", entry.Cid) } - entry = <-entries - if entry.Err != nil { - t.Fatal(entry.Err) + entry, ok = <-entries + if !ok { + t.Fatal("expected another entry") } if entry.Type != coreiface.TSymlink { t.Errorf("wrong type %s", entry.Type) @@ -716,11 +717,12 @@ func (tp *TestSuite) TestLs(t *testing.T) { t.Errorf("expected symlink target to be /foo/bar, got %s", entry.Target) } - if l, ok := <-entries; ok { - t.Errorf("didn't expect a second link") - if l.Err != nil { - t.Error(l.Err) - } + _, ok = <-entries + if ok { + t.Errorf("didn't expect a another link") + } + if err = <-errCh; err != nil { + t.Error(err) } } @@ -779,13 +781,22 @@ func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, p) - if err != nil { + errCh := make(chan error, 1) + links := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, p, links) + }() + + var count int + for range links { + count++ + } + if err = <-errCh; err != nil { t.Fatal(err) } - if len(links) != 0 { - t.Fatalf("expected 0 links, got %d", len(links)) + if count != 0 { + t.Fatalf("expected 0 links, got %d", count) } } @@ -808,13 +819,22 @@ func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, path.FromCid(nd.Cid())) - if err != nil { + errCh := make(chan error, 1) + links := make(chan coreiface.DirEntry) + go func() { + errCh <- api.Unixfs().Ls(ctx, path.FromCid(nd.Cid()), links) + }() + + var count int + for range links { + count++ + } + if err = <-errCh; err != nil { t.Fatal(err) } - if len(links) != 0 { - t.Fatalf("expected 0 links, got %d", len(links)) + if count != 0 { + t.Fatalf("expected 0 links, got %d", count) } } diff --git a/core/coreiface/unixfs.go b/core/coreiface/unixfs.go index c0150bd12c6..10371998c20 100644 --- a/core/coreiface/unixfs.go +++ b/core/coreiface/unixfs.go @@ -2,6 +2,7 @@ package iface import ( "context" + "iter" "os" "time" @@ -63,8 +64,6 @@ type DirEntry struct { Mode os.FileMode ModTime time.Time - - Err error } // UnixfsAPI is the basic interface to immutable files in IPFS @@ -81,7 +80,56 @@ type UnixfsAPI interface { // to operations performed on the returned file Get(context.Context, path.Path) (files.Node, error) - // Ls returns the list of links in a directory. Links aren't guaranteed to be - // returned in order - Ls(context.Context, path.Path, ...options.UnixfsLsOption) (<-chan DirEntry, error) + // Ls writes the links in a directory to the DirEntry channel. Links aren't + // guaranteed to be returned in order. If an error occurs or the context is + // canceled, the DirEntry channel is closed and an error is returned. + // + // Example: + // + // dirs := make(chan DirEntry) + // lsErr := make(chan error, 1) + // go func() { + // lsErr <- Ls(ctx, p, dirs) + // }() + // for dirEnt := range dirs { + // fmt.Println("Dir name:", dirEnt.Name) + // } + // err := <-lsErr + // if err != nil { + // return fmt.Errorf("error listing directory: %w", err) + // } + Ls(context.Context, path.Path, chan<- DirEntry, ...options.UnixfsLsOption) error +} + +// LsIter returns a go iterator that allows ranging over DirEntry results. +// Iteration stops if the context is canceled or if the iterator yields an +// error. +// +// Exmaple: +// +// for dirEnt, err := LsIter(ctx, ufsAPI, p) { +// if err != nil { +// return fmt.Errorf("error listing directory: %w", err) +// } +// fmt.Println("Dir name:", dirEnt.Name) +// } +func LsIter(ctx context.Context, api UnixfsAPI, p path.Path, opts ...options.UnixfsLsOption) iter.Seq2[DirEntry, error] { + return func(yield func(DirEntry, error) bool) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() // cancel Ls if done iterating early + + dirs := make(chan DirEntry) + lsErr := make(chan error, 1) + go func() { + lsErr <- api.Ls(ctx, p, dirs, opts...) + }() + for dirEnt := range dirs { + if !yield(dirEnt, nil) { + return + } + } + if err := <-lsErr; err != nil { + yield(DirEntry{}, err) + } + } } diff --git a/core/node/bitswap.go b/core/node/bitswap.go index 4132d5a0186..d379a5866ce 100644 --- a/core/node/bitswap.go +++ b/core/node/bitswap.go @@ -5,9 +5,12 @@ import ( "time" "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/client" "github.com/ipfs/boxo/bitswap/network" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" + "github.com/ipfs/boxo/exchange/providing" + provider "github.com/ipfs/boxo/provider" "github.com/ipfs/kubo/config" irouting "github.com/ipfs/kubo/routing" "github.com/libp2p/go-libp2p/core/host" @@ -34,7 +37,7 @@ type bitswapOptionsOut struct { // BitswapOptions creates configuration options for Bitswap from the config file // and whether to provide data. -func BitswapOptions(cfg *config.Config, provide bool) interface{} { +func BitswapOptions(cfg *config.Config) interface{} { return func() bitswapOptionsOut { var internalBsCfg config.InternalBitswap if cfg.Internal.Bitswap != nil { @@ -42,7 +45,6 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} { } opts := []bitswap.Option{ - bitswap.ProvideEnabled(provide), bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))), bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))), @@ -55,7 +57,7 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} { } } -type onlineExchangeIn struct { +type bitswapIn struct { fx.In Mctx helpers.MetricsCtx @@ -65,19 +67,62 @@ type onlineExchangeIn struct { BitswapOpts []bitswap.Option `group:"bitswap-options"` } -// OnlineExchange creates new LibP2P backed block exchange (BitSwap). +// Bitswap creates the BitSwap server/client instance. // Additional options to bitswap.New can be provided via the "bitswap-options" // group. -func OnlineExchange() interface{} { - return func(in onlineExchangeIn, lc fx.Lifecycle) exchange.Interface { - bitswapNetwork := network.NewFromIpfsHost(in.Host, in.Rt) +func Bitswap(provide bool) interface{} { + return func(in bitswapIn, lc fx.Lifecycle) *bitswap.Bitswap { + bitswapNetwork := network.NewFromIpfsHost(in.Host) + + var provider client.ProviderFinder + if provide { + provider = in.Rt + } + bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, in.Bs, in.BitswapOpts...) - exch := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, in.Bs, in.BitswapOpts...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - return exch.Close() + return bs.Close() }, }) + return bs + } +} + +// OnlineExchange creates new LibP2P backed block exchange. +func OnlineExchange() interface{} { + return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface { + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return in.Close() + }, + }) + return in + } +} + +type providingExchangeIn struct { + fx.In + + BaseExch exchange.Interface + Provider provider.System +} + +// ProvidingExchange creates a providing.Exchange with the existing exchange +// and the provider.System. +// We cannot do this in OnlineExchange because it causes cycles so this is for +// a decorator. +func ProvidingExchange(provide bool) interface{} { + return func(in providingExchangeIn, lc fx.Lifecycle) exchange.Interface { + exch := in.BaseExch + if provide { + exch = providing.New(in.BaseExch, in.Provider) + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return exch.Close() + }, + }) + } return exch } } diff --git a/core/node/groups.go b/core/node/groups.go index c74d0be8a2c..519cbb47d6e 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -293,8 +293,11 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part shouldBitswapProvide := !cfg.Experimental.StrategicProviding return fx.Options( - fx.Provide(BitswapOptions(cfg, shouldBitswapProvide)), + fx.Provide(BitswapOptions(cfg)), + fx.Provide(Bitswap(shouldBitswapProvide)), fx.Provide(OnlineExchange()), + // Replace our Exchange with a Providing exchange! + fx.Decorate(ProvidingExchange(shouldBitswapProvide)), fx.Provide(DNSResolver), fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))), fx.Provide(Peering), diff --git a/core/node/libp2p/rcmgr.go b/core/node/libp2p/rcmgr.go index 80bfec34ae6..977461e04bd 100644 --- a/core/node/libp2p/rcmgr.go +++ b/core/node/libp2p/rcmgr.go @@ -7,6 +7,10 @@ import ( "os" "path/filepath" + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core/node/helpers" + "github.com/ipfs/kubo/repo" + "github.com/benbjohnson/clock" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" @@ -16,10 +20,6 @@ import ( rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/multiformats/go-multiaddr" "go.uber.org/fx" - - "github.com/ipfs/kubo/config" - "github.com/ipfs/kubo/core/node/helpers" - "github.com/ipfs/kubo/repo" ) var rcmgrLogger = logging.Logger("rcmgr") @@ -70,7 +70,6 @@ filled in with autocomputed defaults.`) } ropts := []rcmgr.Option{ - rcmgr.WithMetrics(createRcmgrMetrics()), rcmgr.WithTraceReporter(str), rcmgr.WithLimitPerSubnet( nil, diff --git a/core/node/libp2p/rcmgr_metrics.go b/core/node/libp2p/rcmgr_metrics.go deleted file mode 100644 index f8b1a7daa3b..00000000000 --- a/core/node/libp2p/rcmgr_metrics.go +++ /dev/null @@ -1,251 +0,0 @@ -package libp2p - -import ( - "errors" - "strconv" - - "github.com/libp2p/go-libp2p/core/network" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/protocol" - rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" - - "github.com/prometheus/client_golang/prometheus" -) - -func mustRegister(c prometheus.Collector) { - err := prometheus.Register(c) - are := prometheus.AlreadyRegisteredError{} - if errors.As(err, &are) { - return - } - if err != nil { - panic(err) - } -} - -func createRcmgrMetrics() rcmgr.MetricsReporter { - const ( - direction = "direction" - usesFD = "usesFD" - protocol = "protocol" - service = "service" - ) - - connAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_conns_allowed_total", - Help: "allowed connections", - }, - []string{direction, usesFD}, - ) - mustRegister(connAllowed) - - connBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_conns_blocked_total", - Help: "blocked connections", - }, - []string{direction, usesFD}, - ) - mustRegister(connBlocked) - - streamAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_streams_allowed_total", - Help: "allowed streams", - }, - []string{direction}, - ) - mustRegister(streamAllowed) - - streamBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_streams_blocked_total", - Help: "blocked streams", - }, - []string{direction}, - ) - mustRegister(streamBlocked) - - peerAllowed := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_peers_allowed_total", - Help: "allowed peers", - }) - mustRegister(peerAllowed) - - peerBlocked := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_peer_blocked_total", - Help: "blocked peers", - }) - mustRegister(peerBlocked) - - protocolAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_allowed_total", - Help: "allowed streams attached to a protocol", - }, - []string{protocol}, - ) - mustRegister(protocolAllowed) - - protocolBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_blocked_total", - Help: "blocked streams attached to a protocol", - }, - []string{protocol}, - ) - mustRegister(protocolBlocked) - - protocolPeerBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_protocols_for_peer_blocked_total", - Help: "blocked streams attached to a protocol for a specific peer", - }, - []string{protocol}, - ) - mustRegister(protocolPeerBlocked) - - serviceAllowed := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_services_allowed_total", - Help: "allowed streams attached to a service", - }, - []string{service}, - ) - mustRegister(serviceAllowed) - - serviceBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_services_blocked_total", - Help: "blocked streams attached to a service", - }, - []string{service}, - ) - mustRegister(serviceBlocked) - - servicePeerBlocked := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "libp2p_rcmgr_service_for_peer_blocked_total", - Help: "blocked streams attached to a service for a specific peer", - }, - []string{service}, - ) - mustRegister(servicePeerBlocked) - - memoryAllowed := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_memory_allocations_allowed_total", - Help: "allowed memory allocations", - }) - mustRegister(memoryAllowed) - - memoryBlocked := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "libp2p_rcmgr_memory_allocations_blocked_total", - Help: "blocked memory allocations", - }) - mustRegister(memoryBlocked) - - return rcmgrMetrics{ - connAllowed, - connBlocked, - streamAllowed, - streamBlocked, - peerAllowed, - peerBlocked, - protocolAllowed, - protocolBlocked, - protocolPeerBlocked, - serviceAllowed, - serviceBlocked, - servicePeerBlocked, - memoryAllowed, - memoryBlocked, - } -} - -// Failsafe to ensure interface from go-libp2p-resource-manager is implemented -var _ rcmgr.MetricsReporter = rcmgrMetrics{} - -type rcmgrMetrics struct { - connAllowed *prometheus.CounterVec - connBlocked *prometheus.CounterVec - streamAllowed *prometheus.CounterVec - streamBlocked *prometheus.CounterVec - peerAllowed prometheus.Counter - peerBlocked prometheus.Counter - protocolAllowed *prometheus.CounterVec - protocolBlocked *prometheus.CounterVec - protocolPeerBlocked *prometheus.CounterVec - serviceAllowed *prometheus.CounterVec - serviceBlocked *prometheus.CounterVec - servicePeerBlocked *prometheus.CounterVec - memoryAllowed prometheus.Counter - memoryBlocked prometheus.Counter -} - -func getDirection(d network.Direction) string { - switch d { - default: - return "" - case network.DirInbound: - return "inbound" - case network.DirOutbound: - return "outbound" - } -} - -func (r rcmgrMetrics) AllowConn(dir network.Direction, usefd bool) { - r.connAllowed.WithLabelValues(getDirection(dir), strconv.FormatBool(usefd)).Inc() -} - -func (r rcmgrMetrics) BlockConn(dir network.Direction, usefd bool) { - r.connBlocked.WithLabelValues(getDirection(dir), strconv.FormatBool(usefd)).Inc() -} - -func (r rcmgrMetrics) AllowStream(_ peer.ID, dir network.Direction) { - r.streamAllowed.WithLabelValues(getDirection(dir)).Inc() -} - -func (r rcmgrMetrics) BlockStream(_ peer.ID, dir network.Direction) { - r.streamBlocked.WithLabelValues(getDirection(dir)).Inc() -} - -func (r rcmgrMetrics) AllowPeer(_ peer.ID) { - r.peerAllowed.Inc() -} - -func (r rcmgrMetrics) BlockPeer(_ peer.ID) { - r.peerBlocked.Inc() -} - -func (r rcmgrMetrics) AllowProtocol(proto protocol.ID) { - r.protocolAllowed.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) BlockProtocol(proto protocol.ID) { - r.protocolBlocked.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) BlockProtocolPeer(proto protocol.ID, _ peer.ID) { - r.protocolPeerBlocked.WithLabelValues(string(proto)).Inc() -} - -func (r rcmgrMetrics) AllowService(svc string) { - r.serviceAllowed.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) BlockService(svc string) { - r.serviceBlocked.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) BlockServicePeer(svc string, _ peer.ID) { - r.servicePeerBlocked.WithLabelValues(svc).Inc() -} - -func (r rcmgrMetrics) AllowMemory(_ int) { - r.memoryAllowed.Inc() -} - -func (r rcmgrMetrics) BlockMemory(_ int) { - r.memoryBlocked.Inc() -} diff --git a/core/node/provider.go b/core/node/provider.go index 7f37cd8b1a4..d6ab26b0c0d 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -14,8 +14,12 @@ import ( "go.uber.org/fx" ) +// The size of a batch that will be used for calculating average announcement +// time per CID, inside of boxo/provider.ThroughputReport +// and in 'ipfs stats provide' report. +const sampledBatchSize = 1000 + func ProviderSys(reprovideInterval time.Duration, acceleratedDHTClient bool) fx.Option { - const magicThroughputReportCount = 128 return fx.Provide(func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, keyProvider provider.KeyChanFunc, repo repo.Repo, bs blockstore.Blockstore) (provider.System, error) { opts := []provider.Option{ provider.Online(cr), @@ -105,7 +109,7 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli keysProvided, avgProvideSpeed, count, avgProvideSpeed*time.Duration(count), reprovideInterval) } return false - }, magicThroughputReportCount)) + }, sampledBatchSize)) } sys, err := provider.New(repo.Datastore(), opts...) if err != nil { diff --git a/docs/add-code-flow.md b/docs/add-code-flow.md index a13c7177d40..264731a3b13 100644 --- a/docs/add-code-flow.md +++ b/docs/add-code-flow.md @@ -1,6 +1,6 @@ # IPFS : The `Add` command demystified -The goal of this document is to capture the code flow for adding a file (see the `coreapi` package) using the IPFS CLI, in the process exploring some datastructures and packages like `ipld.Node` (aka `dagnode`), `FSNode`, `MFS`, etc. +The goal of this document is to capture the code flow for adding a file (see the `coreapi` package) using the IPFS CLI, in the process exploring some data structures and packages like `ipld.Node` (aka `dagnode`), `FSNode`, `MFS`, etc. ## Concepts - [Files](https://github.com/ipfs/docs/issues/133) @@ -99,4 +99,4 @@ Within the function, a new `Adder` is created with the configured `Blockstore` a - **[`adder.PinRoot()`](https://github.com/ipfs/go-ipfs/blob/v0.4.18/core/coreunix/add.go#L171)** - *Pin all files under the `MFS` **root*** - The whole process ends with `PinRoot` recursively pinning all the files under the `MFS` **root** \ No newline at end of file + The whole process ends with `PinRoot` recursively pinning all the files under the `MFS` **root** diff --git a/docs/changelogs/v0.32.md b/docs/changelogs/v0.32.md index 790390978ee..f00cca611a7 100644 --- a/docs/changelogs/v0.32.md +++ b/docs/changelogs/v0.32.md @@ -1,6 +1,6 @@ # Kubo changelog v0.32 -- [v0.32.0](#v0310) +- [v0.32.0](#v0320) ## v0.32.0 diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md index 711092a2cfd..99d7109124f 100644 --- a/docs/changelogs/v0.33.md +++ b/docs/changelogs/v0.33.md @@ -6,6 +6,10 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Shared TCP listeners](#shared-tcp-listeners) + - [Bitswap improvements from Boxo](#bitswap-improvements-from-boxo) + - [Using default `libp2p_rcmgr` metrics](#using-default-libp2p_rcmgr--metrics) + - [`ipfs add --to-files` no longer works with `--wrap`](#ipfs-add---to-files-no-longer-works-with---wrap) - [๐Ÿ“ฆ๏ธ Dependency updates](#-dependency-updates) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -14,8 +18,31 @@ ### ๐Ÿ”ฆ Highlights +#### Shared TCP listeners + +Kubo now supports sharing the same port (`4001` by default) by multiple libp2p transports, such as [raw TCP](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworktcp) and [WebSockets](https://github.com/ipfs/kubo/blob/master/docs/config.md#swarmtransportsnetworkwebsocket). + +#### Bitswap improvements from Boxo + +This release includes some refactorings and improvements affecting Bitswap which should improve reliability. One of the changes affects blocks providing. Previously, the bitswap layer took care itself of announcing new blocks -added or received- with the configured provider (i.e. DHT). This bypassed the "Reprovider", that is, the system that manages precisely "providing" the blocks stored by Kubo. The Reprovider knows how to take advantage of the [AcceleratedDHTClient](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient), is able to handle priorities, logs statistics and is able to resume on daemon reboot where it left off. From now on, Bitswap will not be doing any providing on-the-side and all announcements are managed by the reprovider. In some cases, when the reproviding queue is full with other elements, this may cause additional delays, but more likely this will result in improved block-providing behaviour overall. + +#### Using default `libp2p_rcmgr` metrics + +Bespoke rcmgr metrics [were removed](https://github.com/ipfs/kubo/pull/9947), Kubo now exposes only the default `libp2p_rcmgr` metrics from go-libp2p. +This makes it easier to compare Kubo with custom implementations based on go-libp2p. +If you depended on removed ones, please fill an issue to add them to the upstream [go-libp2p](https://github.com/libp2p/go-libp2p). + +#### `ipfs add --to-files` no longer works with `--wrap` + +Onboarding files and directories with `ipfs add --to-files` now requires non-empty names. due to this, The `--to-files` and `--wrap` options are now mutually exclusive ([#10612](https://github.com/ipfs/kubo/issues/10612)). + #### ๐Ÿ“ฆ๏ธ Dependency updates +- update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO) +- update `go-libp2p` to [v0.37.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.1) + [v0.37.2](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.2) +- update `p2p-forge/client` to [v0.1.0](https://github.com/ipshipyard/p2p-forge/releases/tag/v0.1.0) +- update `ipfs-webui` to [v4.4.1](https://github.com/ipfs/ipfs-webui/releases/tag/v4.4.1) + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index 7c4e9e306d4..ba99711b0f5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -496,15 +496,15 @@ Enables AutoTLS feature to get DNS+TLS for [libp2p Secure WebSocket](https://git If `.../tls/sni/*.libp2p.direct/ws` [multiaddr] is present in [`Addresses.Swarm`](#addressesswarm) with SNI segment ending with [`AutoTLS.DomainSuffix`](#autotlsdomainsuffix), -Kubo will obtain and set up a trusted PKI TLS certificate for it, making it diallable from web browser's [Secure Contexts](https://w3c.github.io/webappsec-secure-contexts/). +Kubo will obtain and set up a trusted PKI TLS certificate for it, making it dialable from web browser's [Secure Contexts](https://w3c.github.io/webappsec-secure-contexts/). > [!IMPORTANT] > Caveats: -> - Requires your Kubo node to be publicly diallable. +> - Requires your Kubo node to be publicly dialable. > - If you want to test this with a node that is behind a NAT and uses manual port forwarding or UPnP (`Swarm.DisableNatPortMap=false`), > add catch-all `/ip4/0.0.0.0/tcp/4001/tls/sni/*.libp2p.direct/ws` and `/ip6/::/tcp/4001/tls/sni/*.libp2p.direct/ws` to [`Addresses.Swarm`](#addressesswarm) > and **wait 5-15 minutes** for libp2p node to set up and learn about own public addresses via [AutoNAT](#autonat). -> - If your node is fresh and just started, the [p2p-forge] client may produce and log ERRORs during this time, but once a publicly diallable addresses are set up, a subsequent retry should be successful. +> - If your node is fresh and just started, the [p2p-forge] client may produce and log ERRORs during this time, but once a publicly dialable addresses are set up, a subsequent retry should be successful. > - The TLS certificate is used only for [libp2p WebSocket](https://github.com/libp2p/specs/blob/master/websockets/README.md) connections. > - Right now, this is NOT used for hosting a [Gateway](#gateway) over HTTPS (that use case still requires manual TLS setup on reverse proxy, and your own domain). @@ -533,7 +533,7 @@ Do not change this unless you self-host [p2p-forge] under own domain. > [!IMPORTANT] > The default endpoint performs [libp2p Peer ID Authentication over HTTP](https://github.com/libp2p/specs/blob/master/http/peer-id-auth.md) > (proving ownership of PeerID), probes if your Kubo node can correctly answer to a [libp2p Identify](https://github.com/libp2p/specs/tree/master/identify) query. -> This ensures only a correctly configured, publicly diallable Kubo can initiate [ACME DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for `peerid.libp2p.direct`. +> This ensures only a correctly configured, publicly dialable Kubo can initiate [ACME DNS-01 challenge](https://letsencrypt.org/docs/challenge-types/#dns-01-challenge) for `peerid.libp2p.direct`. Default: `https://registration.libp2p.direct` (public good run by [Interplanetary Shipyard](https://ipshipyard.com)) @@ -564,7 +564,7 @@ Type: `optionalString` Bootstrap is an array of [multiaddrs][multiaddr] of trusted nodes that your node connects to, to fetch other nodes of the network on startup. -Default: The ipfs.io bootstrap nodes +Default: [`config.DefaultBootstrapAddresses`](https://github.com/ipfs/kubo/blob/master/config/bootstrap_peers.go) Type: `array[string]` ([multiaddrs][multiaddr]) diff --git a/docs/datastores.md b/docs/datastores.md index f2051b601d4..321f59bfc8d 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -53,11 +53,11 @@ Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store. } ``` -The following options are availble for tuning pebble. +The following options are available for tuning pebble. If they are not configured (or assigned their zero-valued), then default values are used. * `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB) -* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) +* `disableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false) * `cacheSize`: Size of pebble's shared block cache. (default: 8MB) * `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction. * `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels. diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9b40822164f..6cdc6b9e143 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/boxo v0.24.4-0.20241203185533-3a3e8afa3492 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 - github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e + github.com/libp2p/go-libp2p v0.37.2 github.com/multiformats/go-multiaddr v0.13.0 ) @@ -111,7 +111,7 @@ require ( github.com/ipld/go-car/v2 v2.14.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect - github.com/ipshipyard/p2p-forge v0.0.2 // indirect + github.com/ipshipyard/p2p-forge v0.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -240,3 +240,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect ) + +replace github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 676bfe8cb49..1c5a2b8f056 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -405,8 +405,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -734,8 +734,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= -github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= +github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= +github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1078,8 +1078,8 @@ golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/go.mod b/go.mod index 28303ba419c..2090b3811e0 100644 --- a/go.mod +++ b/go.mod @@ -49,12 +49,12 @@ require ( github.com/ipld/go-car/v2 v2.14.2 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.21.0 - github.com/ipshipyard/p2p-forge v0.0.2 + github.com/ipshipyard/p2p-forge v0.1.0 github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 github.com/julienschmidt/httprouter v1.3.0 github.com/libp2p/go-doh-resolver v0.4.0 - github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e + github.com/libp2p/go-libp2p v0.37.2 github.com/libp2p/go-libp2p-http v0.5.0 github.com/libp2p/go-libp2p-kad-dht v0.28.1 github.com/libp2p/go-libp2p-kbucket v0.6.4 @@ -224,7 +224,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.60.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/statsd_exporter v0.26.1 // indirect + github.com/prometheus/statsd_exporter v0.27.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect @@ -233,7 +233,7 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/rs/cors v1.11.1 // indirect github.com/samber/lo v1.47.0 // indirect - github.com/slok/go-http-metrics v0.12.0 // indirect + github.com/slok/go-http-metrics v0.13.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/texttheater/golang-levenshtein v1.0.1 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -274,3 +274,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect ) + +replace github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e diff --git a/go.sum b/go.sum index 004a82ee0ba..a152916af7a 100644 --- a/go.sum +++ b/go.sum @@ -473,8 +473,8 @@ github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= @@ -816,8 +816,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/prometheus/statsd_exporter v0.26.1 h1:ucbIAdPmwAUcA+dU+Opok8Qt81Aw8HanlO+2N/Wjv7w= -github.com/prometheus/statsd_exporter v0.26.1/go.mod h1:XlDdjAmRmx3JVvPPYuFNUg+Ynyb5kR69iPPkQjxXFMk= +github.com/prometheus/statsd_exporter v0.27.1 h1:tcRJOmwlA83HPfWzosAgr2+zEN5XDFv+M2mn/uYkn5Y= +github.com/prometheus/statsd_exporter v0.27.1/go.mod h1:vA6ryDfsN7py/3JApEst6nLTJboq66XsNcJGNmC88NQ= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= @@ -868,8 +868,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= -github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= +github.com/slok/go-http-metrics v0.13.0 h1:lQDyJJx9wKhmbliyUsZ2l6peGnXRHjsjoqPt5VYzcP8= +github.com/slok/go-http-metrics v0.13.0/go.mod h1:HIr7t/HbN2sJaunvnt9wKP9xoBBVZFo1/KiHU3b0w+4= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1292,8 +1292,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/test/cli/delegated_routing_v1_http_proxy_test.go b/test/cli/delegated_routing_v1_http_proxy_test.go index 1d80ae50a5f..ef3bc4fe2bc 100644 --- a/test/cli/delegated_routing_v1_http_proxy_test.go +++ b/test/cli/delegated_routing_v1_http_proxy_test.go @@ -15,9 +15,11 @@ func TestRoutingV1Proxy(t *testing.T) { t.Parallel() setupNodes := func(t *testing.T) harness.Nodes { - nodes := harness.NewT(t).NewNodes(2).Init() + nodes := harness.NewT(t).NewNodes(3).Init() - // Node 0 uses DHT and exposes the Routing API. + // Node 0 uses DHT and exposes the Routing API. For the DHT + // to actually work there will need to be another DHT-enabled + // node. nodes[0].UpdateConfig(func(cfg *config.Config) { cfg.Gateway.ExposeRoutingAPI = config.True cfg.Discovery.MDNS.Enabled = false @@ -49,6 +51,15 @@ func TestRoutingV1Proxy(t *testing.T) { }) nodes[1].StartDaemon() + // This is the second DHT node. Only used so that the DHT is + // operative. + nodes[2].UpdateConfig(func(cfg *config.Config) { + cfg.Gateway.ExposeRoutingAPI = config.True + cfg.Discovery.MDNS.Enabled = false + cfg.Routing.Type = config.NewOptionalString("dht") + }) + nodes[2].StartDaemon() + // Connect them. nodes.Connect() @@ -60,8 +71,10 @@ func TestRoutingV1Proxy(t *testing.T) { nodes := setupNodes(t) cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000)) - - res := nodes[1].IPFS("routing", "findprovs", cidStr) + // Reprovide as initialProviderDelay still ongoing + res := nodes[0].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) + res = nodes[1].IPFS("routing", "findprovs", cidStr) assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed()) }) diff --git a/test/cli/delegated_routing_v1_http_server_test.go b/test/cli/delegated_routing_v1_http_server_test.go index f2bd98cb77b..916b9188280 100644 --- a/test/cli/delegated_routing_v1_http_server_test.go +++ b/test/cli/delegated_routing_v1_http_server_test.go @@ -14,6 +14,7 @@ import ( "github.com/ipfs/kubo/test/cli/harness" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRoutingV1Server(t *testing.T) { @@ -38,6 +39,9 @@ func TestRoutingV1Server(t *testing.T) { text := "hello world " + uuid.New().String() cidStr := nodes[2].IPFSAddStr(text) _ = nodes[3].IPFSAddStr(text) + // Reprovide as initialProviderDelay still ongoing + res := nodes[3].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) cid, err := cid.Decode(cidStr) assert.NoError(t, err) diff --git a/test/cli/harness/run.go b/test/cli/harness/run.go index 8ca85eb63b1..077af6ca574 100644 --- a/test/cli/harness/run.go +++ b/test/cli/harness/run.go @@ -3,6 +3,7 @@ package harness import ( "fmt" "io" + "os" "os/exec" "strings" ) @@ -60,8 +61,27 @@ func environToMap(environ []string) map[string]string { func (r *Runner) Run(req RunRequest) *RunResult { cmd := exec.Command(req.Path, req.Args...) - stdout := &Buffer{} - stderr := &Buffer{} + var stdout io.Writer + var stderr io.Writer + outbuf := &Buffer{} + errbuf := &Buffer{} + + if r.Verbose { + or, ow := io.Pipe() + errr, errw := io.Pipe() + stdout = io.MultiWriter(outbuf, ow) + stderr = io.MultiWriter(errbuf, errw) + go func() { + _, _ = io.Copy(os.Stdout, or) + }() + go func() { + _, _ = io.Copy(os.Stderr, errr) + }() + } else { + stdout = outbuf + stderr = errbuf + } + cmd.Stdout = stdout cmd.Stderr = stderr cmd.Dir = r.Dir @@ -83,8 +103,8 @@ func (r *Runner) Run(req RunRequest) *RunResult { err := req.RunFunc(cmd) result := RunResult{ - Stdout: stdout, - Stderr: stderr, + Stdout: outbuf, + Stderr: errbuf, Cmd: cmd, Err: err, } diff --git a/test/cli/provider_test.go b/test/cli/provider_test.go index 5ecf8f3cab7..546ac3fd711 100644 --- a/test/cli/provider_test.go +++ b/test/cli/provider_test.go @@ -42,6 +42,9 @@ func TestProvider(t *testing.T) { defer nodes.StopDaemons() cid := nodes[0].IPFSAddStr(time.Now().String()) + // Reprovide as initialProviderDelay still ongoing + res := nodes[0].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...) }) diff --git a/test/cli/routing_dht_test.go b/test/cli/routing_dht_test.go index fb0d391951e..d149e93a2a3 100644 --- a/test/cli/routing_dht_test.go +++ b/test/cli/routing_dht_test.go @@ -84,7 +84,10 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) { t.Run("ipfs routing findprovs", func(t *testing.T) { t.Parallel() hash := nodes[3].IPFSAddStr("some stuff") - res := nodes[4].IPFS("routing", "findprovs", hash) + // Reprovide as initialProviderDelay still ongoing + res := nodes[3].IPFS("bitswap", "reprovide") + require.NoError(t, res.Err) + res = nodes[4].IPFS("routing", "findprovs", hash) assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed()) }) diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 22ae0b0463f..90285545cda 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -130,7 +130,7 @@ require ( github.com/ipfs/kubo v0.31.0 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect - github.com/ipshipyard/p2p-forge v0.0.2 // indirect + github.com/ipshipyard/p2p-forge v0.1.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -156,7 +156,7 @@ require ( github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect - github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e // indirect + github.com/libp2p/go-libp2p v0.37.2 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect @@ -313,3 +313,5 @@ require ( mvdan.cc/gofumpt v0.7.0 // indirect mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect ) + +replace github.com/libp2p/go-libp2p => github.com/libp2p/go-libp2p v0.37.1-0.20241202220543-9024f8e8c86e diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 4be30337bca..57fa85a48d6 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -362,8 +362,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= -github.com/ipshipyard/p2p-forge v0.0.2 h1:86y9LxGB8sGxYQ/If5sNx+c8C/huSpBUg3UZ1uvtym8= -github.com/ipshipyard/p2p-forge v0.0.2/go.mod h1:taPeh3PDSO8Ual0/N2tIOAUXPV8gZoPF3uPXoUyiq14= +github.com/ipshipyard/p2p-forge v0.1.0 h1:RjCuX5wSKOv6J+6aaKTvuGOhVw24TuCLZx7d3M4BaiI= +github.com/ipshipyard/p2p-forge v0.1.0/go.mod h1:5s1MuHMh8FXrhDScKLk0F+zBaJglCAZMKn9myiWAPOU= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index 142ab8ec18c..c7232bedd8c 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -355,10 +355,10 @@ test_add_cat_file() { test_cmp expected actual ' - test_must_fail "ipfs add with multiple files of same name but different dirs fails" ' + test_expect_success "ipfs add with multiple files of same name but different dirs fails" ' mkdir -p mountdir/same-file/ && cp mountdir/hello.txt mountdir/same-file/hello.txt && - ipfs add mountdir/hello.txt mountdir/same-file/hello.txt >actual && + test_expect_code 1 ipfs add mountdir/hello.txt mountdir/same-file/hello.txt >actual && rm mountdir/same-file/hello.txt && rmdir mountdir/same-file ' @@ -469,6 +469,15 @@ test_add_cat_file() { ipfs files rm -r --force /mfs ' + # confirm -w and --to-files are exclusive + # context: https://github.com/ipfs/kubo/issues/10611 + test_expect_success "ipfs add -r -w dir --to-files /mfs/subdir5/ errors (-w and --to-files are exclusive)" ' + ipfs files mkdir -p /mfs/subdir5 && + test_expect_code 1 ipfs add -r -w test --to-files /mfs/subdir5/ >actual 2>&1 && + test_should_contain "Error" actual && + ipfs files rm -r --force /mfs + ' + } test_add_cat_5MB() { diff --git a/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr b/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr index 382ab125602..e69de29bb2d 100644 --- a/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr +++ b/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_enabling_rcmgr @@ -1,4 +0,0 @@ -libp2p_rcmgr_memory_allocations_allowed_total -libp2p_rcmgr_memory_allocations_blocked_total -libp2p_rcmgr_peer_blocked_total -libp2p_rcmgr_peers_allowed_total diff --git a/test/sharness/t0120-bootstrap.sh b/test/sharness/t0120-bootstrap.sh index 2922533c628..00141da1f06 100755 --- a/test/sharness/t0120-bootstrap.sh +++ b/test/sharness/t0120-bootstrap.sh @@ -9,8 +9,9 @@ BP1="/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTez BP2="/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa" BP3="/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb" BP4="/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt" -BP5="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" -BP6="/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" +BP5="/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8" +BP6="/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" +BP7="/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" test_description="Test ipfs repo operations" @@ -93,10 +94,11 @@ test_bootstrap_cmd() { echo "added $BP4" >>add2_expected && echo "added $BP5" >>add2_expected && echo "added $BP6" >>add2_expected && + echo "added $BP7" >>add2_expected && test_cmp add2_expected add2_actual ' - test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 $BP6 + test_bootstrap_list_cmd $BP1 $BP2 $BP3 $BP4 $BP5 $BP6 $BP7 test_expect_success "'ipfs bootstrap rm --all' succeeds" ' ipfs bootstrap rm --all >rm2_actual @@ -109,6 +111,7 @@ test_bootstrap_cmd() { echo "removed $BP4" >>rm2_expected && echo "removed $BP5" >>rm2_expected && echo "removed $BP6" >>rm2_expected && + echo "removed $BP7" >>rm2_expected && test_cmp rm2_expected rm2_actual ' diff --git a/test/sharness/t0220-bitswap.sh b/test/sharness/t0220-bitswap.sh index 3575f0d33e2..412437651be 100755 --- a/test/sharness/t0220-bitswap.sh +++ b/test/sharness/t0220-bitswap.sh @@ -18,7 +18,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 @@ -56,7 +55,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" ' test_expect_success "'ipfs bitswap stat' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 @@ -85,7 +83,6 @@ test_expect_success "'ipfs bitswap stat --human' succeeds" ' test_expect_success "'ipfs bitswap stat --human' output looks good" ' cat <expected && bitswap status - provides buffer: 0 / 256 blocks received: 0 blocks sent: 0 data received: 0 B diff --git a/tracing/doc.go b/tracing/doc.go index d442ea2db50..2c9711a63a1 100644 --- a/tracing/doc.go +++ b/tracing/doc.go @@ -6,7 +6,7 @@ // // Tracing is configured through environment variables, as consistent with the OpenTelemetry spec as possible: // -// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md // // OTEL_TRACES_EXPORTER: a comma-separated list of exporters: // - otlp