stackdriver: add min-level and max-value-size

go/config/config.go

@@ -79,6 +79,21 @@ func (c *Config) SetupDefaultLogger() error {
 		if err != nil {
 			return fmt.Errorf("create Stackdriver emitter: %w", err)
 		}
+		if levelStr := os.Getenv("LOGS_STACKDRIVER_MIN_LEVEL"); levelStr != "" {
+			if printer.MinLevel, err = logs.ParseLevel(levelStr); err != nil {
+				return err
+			}
+		}
+		if valStr := os.Getenv("LOGS_STACKDRIVER_MAX_VALUE_SIZE"); valStr != "" {
+			value, err := strconv.Atoi(valStr)
+			if err == nil && value <= 0 {
+				err = fmt.Errorf("non-positive")
+			}
+			if err != nil {
+				return fmt.Errorf("invalid LOGS_STACKDRIVER_MAX_VALUE_SIZE %q: %w", valStr, err)
+			}
+			printer.MaxValueSize = value
+		}
 		emitters = append(emitters, printer)
 	default:
 		return fmt.Errorf("unknown console printer: %s", c.ConsolePrinter)
@@ -87,7 +102,7 @@ func (c *Config) SetupDefaultLogger() error {
 	if c.BlobFile != "" {
 		fn, err := blob.CreateFileWith(c.BlobFile)
 		if err != nil {
-			return fmt.Errorf("Blob filename template: %w", err)
+			return fmt.Errorf("blob filename template: %w", err)
 		}
 		emitters = append(emitters, &blob.Emitter{CreateFile: fn, Sync: c.BlobSync, SizeLimit: c.BlobSizeLimit})
 	}

go/emitters/stackdriver/emitter.go

@@ -15,6 +15,10 @@ import (
 	"github.com/evo-cloud/logs/go/logs"
 )
 
+const (
+	DefaultMaxValueSize = 8192 // 8K.
+)
+
 var (
 	// ErrUnknownProjectID indicate project ID is not provided and can't be determined.
 	ErrUnknownProjectID = errors.New("unknown GCP project id")
@@ -50,6 +54,9 @@ type Timestamp struct {
 type JSONEmitter struct {
 	Out       io.Writer
 	ProjectID string
+	MinLevel  logspb.LogEntry_Level
+	// MaxValueSize applies to the value of a single attribute or the message.
+	MaxValueSize int
 }
 
 // NewJSONEmitter creates a JSONEmitter.
@@ -64,18 +71,27 @@ func NewJSONEmitter(out io.Writer, projectID string) (*JSONEmitter, error) {
 		}
 		projectID = id
 	}
-	return &JSONEmitter{Out: out, ProjectID: projectID}, nil
+	return &JSONEmitter{Out: out, ProjectID: projectID, MaxValueSize: DefaultMaxValueSize}, nil
 }
 
 // EmitLogEntry implements LogEmitter.
 func (e *JSONEmitter) EmitLogEntry(entry *logspb.LogEntry) {
+	if entry.GetLevel() < e.MinLevel {
+		return
+	}
 	payload := &JSONPayload{
 		Timestamp: timestampFromNanos(entry.GetNanoTs()),
 		Severity:  severityFromLevel(entry.GetLevel()),
 		Message:   entry.GetMessage(),
-		Labels:    labelsFromAttributes(entry.GetAttributes()),
+		Labels:    labelsFromAttributes(entry.GetAttributes(), e.MaxValueSize),
 		Raw:       json.RawMessage(protojson.MarshalOptions{UseProtoNames: true}.Format(entry)),
 	}
+	if sz := len(payload.Message); e.MaxValueSize > 0 && sz > e.MaxValueSize {
+		payload.Message = payload.Message[:e.MaxValueSize] + "...<truncated>"
+	}
+	if sz := len(payload.Raw); e.MaxValueSize > 0 && sz > e.MaxValueSize {
+		payload.Raw = json.RawMessage("<truncated>")
+	}
 	loc := strings.SplitN(entry.GetLocation(), ":", 2)
 	if len(loc) > 1 {
 		if line, err := strconv.Atoi(loc[1]); err == nil {
@@ -120,7 +136,7 @@ func severityFromLevel(level logspb.LogEntry_Level) string {
 	return "DEFAULT"
 }
 
-func labelsFromAttributes(attrs map[string]*logspb.Value) map[string]interface{} {
+func labelsFromAttributes(attrs map[string]*logspb.Value, maxValueSize int) map[string]interface{} {
 	if len(attrs) == 0 {
 		return nil
 	}
@@ -138,9 +154,17 @@ func labelsFromAttributes(attrs map[string]*logspb.Value) map[string]interface{}
 		case *logspb.Value_StrValue:
 			labels[key] = v.StrValue
 		case *logspb.Value_Json:
-			labels[key] = json.RawMessage(v.Json)
+			if sz := len(v.Json); maxValueSize > 0 && sz > maxValueSize {
+				labels[key] = "json:<too long...>"
+			} else {
+				labels[key] = json.RawMessage(v.Json)
+			}
 		case *logspb.Value_Proto:
-			labels[key] = v.Proto
+			if sz := len(v.Proto); maxValueSize > 0 && sz > maxValueSize {
+				labels[key] = "pb:<too long...>"
+			} else {
+				labels[key] = v.Proto
+			}
 		default:
 			continue
 		}

go/logs/common.go

@@ -0,0 +1,29 @@
+package logs
+
+import (
+	"fmt"
+	"strings"
+
+	logspb "github.com/evo-cloud/logs/go/gen/proto/logs"
+)
+
+// ParseLevel parses a human friendly level string to log level.
+func ParseLevel(str string) (logspb.LogEntry_Level, error) {
+	level := logspb.LogEntry_NONE
+	switch strings.ToLower(str) {
+	case "", "no", "none":
+	case "i", "info":
+		level = logspb.LogEntry_INFO
+	case "w", "warn", "warning":
+		level = logspb.LogEntry_WARNING
+	case "e", "err", "error":
+		level = logspb.LogEntry_ERROR
+	case "c", "crit", "critical":
+		level = logspb.LogEntry_CRITICAL
+	case "f", "fatal":
+		level = logspb.LogEntry_FATAL
+	default:
+		return level, fmt.Errorf("unknown level: %s", str)
+	}
+	return level, nil
+}

go/source/filter_parser.go

@@ -9,6 +9,7 @@ import (
 	"github.com/jinzhu/now"
 
 	logspb "github.com/evo-cloud/logs/go/gen/proto/logs"
+	"github.com/evo-cloud/logs/go/logs"
 )
 
 // ParseFilters parses a list of filters in strings into LogEntryFilters.
@@ -52,7 +53,7 @@ func ParseFilter(str string) (LogEntryFilter, error) {
 		}
 		return FilterBefore(t), nil
 	case "l", "lv", "level":
-		level, err := parseLevel(val)
+		level, err := logs.ParseLevel(val)
 		if err != nil {
 			return nil, err
 		}
@@ -76,7 +77,7 @@ func ParseFilter(str string) (LogEntryFilter, error) {
 		return nil, nil
 
 	default:
-		return nil, fmt.Errorf("Unknown filter: %s", str)
+		return nil, fmt.Errorf("unknown filter: %s", str)
 	}
 }
 
@@ -91,23 +92,3 @@ func parseTime(str string) (time.Time, error) {
 	}
 	return t, nil
 }
-
-func parseLevel(str string) (logspb.LogEntry_Level, error) {
-	level := logspb.LogEntry_NONE
-	switch strings.ToLower(str) {
-	case "", "no", "none":
-	case "i", "info":
-		level = logspb.LogEntry_INFO
-	case "w", "warn", "warning":
-		level = logspb.LogEntry_WARNING
-	case "e", "err", "error":
-		level = logspb.LogEntry_ERROR
-	case "c", "crit", "critical":
-		level = logspb.LogEntry_CRITICAL
-	case "f", "fatal":
-		level = logspb.LogEntry_FATAL
-	default:
-		return level, fmt.Errorf("unknown level: %s", str)
-	}
-	return level, nil
-}