From 4d109767d2c6f57117e359f8fd23e4e626b267d4 Mon Sep 17 00:00:00 2001 From: EaseWay Date: Fri, 8 Oct 2021 03:57:59 +0000 Subject: [PATCH] stackdriver: add min-level and max-value-size --- go/config/config.go | 17 ++++++++++++++- go/emitters/stackdriver/emitter.go | 34 +++++++++++++++++++++++++----- go/logs/common.go | 29 +++++++++++++++++++++++++ go/source/filter_parser.go | 25 +++------------------- 4 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 go/logs/common.go diff --git a/go/config/config.go b/go/config/config.go index 0286b5c..f51fc1d 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -79,6 +79,21 @@ func (c *Config) SetupDefaultLogger() error { if err != nil { return fmt.Errorf("create Stackdriver emitter: %w", err) } + if levelStr := os.Getenv("LOGS_STACKDRIVER_MIN_LEVEL"); levelStr != "" { + if printer.MinLevel, err = logs.ParseLevel(levelStr); err != nil { + return err + } + } + if valStr := os.Getenv("LOGS_STACKDRIVER_MAX_VALUE_SIZE"); valStr != "" { + value, err := strconv.Atoi(valStr) + if err == nil && value <= 0 { + err = fmt.Errorf("non-positive") + } + if err != nil { + return fmt.Errorf("invalid LOGS_STACKDRIVER_MAX_VALUE_SIZE %q: %w", valStr, err) + } + printer.MaxValueSize = value + } emitters = append(emitters, printer) default: return fmt.Errorf("unknown console printer: %s", c.ConsolePrinter) @@ -87,7 +102,7 @@ func (c *Config) SetupDefaultLogger() error { if c.BlobFile != "" { fn, err := blob.CreateFileWith(c.BlobFile) if err != nil { - return fmt.Errorf("Blob filename template: %w", err) + return fmt.Errorf("blob filename template: %w", err) } emitters = append(emitters, &blob.Emitter{CreateFile: fn, Sync: c.BlobSync, SizeLimit: c.BlobSizeLimit}) } diff --git a/go/emitters/stackdriver/emitter.go b/go/emitters/stackdriver/emitter.go index 93e9c02..a3b28f6 100644 --- a/go/emitters/stackdriver/emitter.go +++ b/go/emitters/stackdriver/emitter.go @@ -15,6 +15,10 @@ import ( "github.com/evo-cloud/logs/go/logs" ) +const ( + DefaultMaxValueSize = 8192 // 8K. +) + var ( // ErrUnknownProjectID indicate project ID is not provided and can't be determined. ErrUnknownProjectID = errors.New("unknown GCP project id") @@ -50,6 +54,9 @@ type Timestamp struct { type JSONEmitter struct { Out io.Writer ProjectID string + MinLevel logspb.LogEntry_Level + // MaxValueSize applies to the value of a single attribute or the message. + MaxValueSize int } // NewJSONEmitter creates a JSONEmitter. @@ -64,18 +71,27 @@ func NewJSONEmitter(out io.Writer, projectID string) (*JSONEmitter, error) { } projectID = id } - return &JSONEmitter{Out: out, ProjectID: projectID}, nil + return &JSONEmitter{Out: out, ProjectID: projectID, MaxValueSize: DefaultMaxValueSize}, nil } // EmitLogEntry implements LogEmitter. func (e *JSONEmitter) EmitLogEntry(entry *logspb.LogEntry) { + if entry.GetLevel() < e.MinLevel { + return + } payload := &JSONPayload{ Timestamp: timestampFromNanos(entry.GetNanoTs()), Severity: severityFromLevel(entry.GetLevel()), Message: entry.GetMessage(), - Labels: labelsFromAttributes(entry.GetAttributes()), + Labels: labelsFromAttributes(entry.GetAttributes(), e.MaxValueSize), Raw: json.RawMessage(protojson.MarshalOptions{UseProtoNames: true}.Format(entry)), } + if sz := len(payload.Message); e.MaxValueSize > 0 && sz > e.MaxValueSize { + payload.Message = payload.Message[:e.MaxValueSize] + "..." + } + if sz := len(payload.Raw); e.MaxValueSize > 0 && sz > e.MaxValueSize { + payload.Raw = json.RawMessage("") + } loc := strings.SplitN(entry.GetLocation(), ":", 2) if len(loc) > 1 { if line, err := strconv.Atoi(loc[1]); err == nil { @@ -120,7 +136,7 @@ func severityFromLevel(level logspb.LogEntry_Level) string { return "DEFAULT" } -func labelsFromAttributes(attrs map[string]*logspb.Value) map[string]interface{} { +func labelsFromAttributes(attrs map[string]*logspb.Value, maxValueSize int) map[string]interface{} { if len(attrs) == 0 { return nil } @@ -138,9 +154,17 @@ func labelsFromAttributes(attrs map[string]*logspb.Value) map[string]interface{} case *logspb.Value_StrValue: labels[key] = v.StrValue case *logspb.Value_Json: - labels[key] = json.RawMessage(v.Json) + if sz := len(v.Json); maxValueSize > 0 && sz > maxValueSize { + labels[key] = "json:" + } else { + labels[key] = json.RawMessage(v.Json) + } case *logspb.Value_Proto: - labels[key] = v.Proto + if sz := len(v.Proto); maxValueSize > 0 && sz > maxValueSize { + labels[key] = "pb:" + } else { + labels[key] = v.Proto + } default: continue } diff --git a/go/logs/common.go b/go/logs/common.go new file mode 100644 index 0000000..d0e451a --- /dev/null +++ b/go/logs/common.go @@ -0,0 +1,29 @@ +package logs + +import ( + "fmt" + "strings" + + logspb "github.com/evo-cloud/logs/go/gen/proto/logs" +) + +// ParseLevel parses a human friendly level string to log level. +func ParseLevel(str string) (logspb.LogEntry_Level, error) { + level := logspb.LogEntry_NONE + switch strings.ToLower(str) { + case "", "no", "none": + case "i", "info": + level = logspb.LogEntry_INFO + case "w", "warn", "warning": + level = logspb.LogEntry_WARNING + case "e", "err", "error": + level = logspb.LogEntry_ERROR + case "c", "crit", "critical": + level = logspb.LogEntry_CRITICAL + case "f", "fatal": + level = logspb.LogEntry_FATAL + default: + return level, fmt.Errorf("unknown level: %s", str) + } + return level, nil +} diff --git a/go/source/filter_parser.go b/go/source/filter_parser.go index 649567d..9f7cf7b 100644 --- a/go/source/filter_parser.go +++ b/go/source/filter_parser.go @@ -9,6 +9,7 @@ import ( "github.com/jinzhu/now" logspb "github.com/evo-cloud/logs/go/gen/proto/logs" + "github.com/evo-cloud/logs/go/logs" ) // ParseFilters parses a list of filters in strings into LogEntryFilters. @@ -52,7 +53,7 @@ func ParseFilter(str string) (LogEntryFilter, error) { } return FilterBefore(t), nil case "l", "lv", "level": - level, err := parseLevel(val) + level, err := logs.ParseLevel(val) if err != nil { return nil, err } @@ -76,7 +77,7 @@ func ParseFilter(str string) (LogEntryFilter, error) { return nil, nil default: - return nil, fmt.Errorf("Unknown filter: %s", str) + return nil, fmt.Errorf("unknown filter: %s", str) } } @@ -91,23 +92,3 @@ func parseTime(str string) (time.Time, error) { } return t, nil } - -func parseLevel(str string) (logspb.LogEntry_Level, error) { - level := logspb.LogEntry_NONE - switch strings.ToLower(str) { - case "", "no", "none": - case "i", "info": - level = logspb.LogEntry_INFO - case "w", "warn", "warning": - level = logspb.LogEntry_WARNING - case "e", "err", "error": - level = logspb.LogEntry_ERROR - case "c", "crit", "critical": - level = logspb.LogEntry_CRITICAL - case "f", "fatal": - level = logspb.LogEntry_FATAL - default: - return level, fmt.Errorf("unknown level: %s", str) - } - return level, nil -}