consumer: clean up shard working directory if NewStore() fails

Previously we failed to clean these up if NewStore() errored out.

consumer/recovery.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 	"time"
 
@@ -219,7 +219,7 @@ func beginRecovery(s *shard) error {
 
 	// Create local temporary directory into which we recover.
 	var dir string
-	if dir, err = ioutil.TempDir("", strings.ReplaceAll(spec.Id.String(), "/", "_")+"-"); err != nil {
+	if dir, err = os.MkdirTemp("", strings.ReplaceAll(spec.Id.String(), "/", "_")+"-"); err != nil {
 		return errors.WithMessage(err, "creating shard working directory")
 	}
 
@@ -278,6 +278,12 @@ func completeRecovery(s *shard) (_ pc.Checkpoint, err error) {
 	}
 
 	if s.store, err = s.svc.App.NewStore(s, s.recovery.recorder); err != nil {
+		if s.store == nil {
+			// s.store is nil, ergo its Destroy() will not be invoked by
+			// waitAndTearDown(), and we are responsible for best-effort
+			// cleanup of the playback directory now.
+			_ = os.RemoveAll(s.recovery.recorder.Dir())
+		}
 		return cp, errors.WithMessage(err, "app.NewStore")
 	} else if cp, err = s.store.RestoreCheckpoint(s); err != nil {
 		return cp, errors.WithMessage(err, "store.RestoreCheckpoint")