v1.0.X-runtime

cmd/observe.go

@@ -30,9 +30,11 @@ var (
 	observePolicyFile   string
 	observeSchedule     string
 	observeReport       string
+	observeMode         string
 	reportMutex         sync.Mutex
 	reportDir           string = "_status"
 	allFileInfos        []FileInfo
+	observeList         []string
 )
 
 var observeCmd = &cobra.Command{
@@ -53,6 +55,7 @@ func init() {
 	observeCmd.Flags().StringVar(&observePolicyFile, "policy", "", "Policy file")
 	observeCmd.Flags().StringVar(&observeSchedule, "schedule", "", "Global Cron Schedule")
 	observeCmd.Flags().StringVar(&observeReport, "report", "", "Report Cron Schedule")
+	observeCmd.Flags().StringVar(&observeMode, "mode", "last", "Observe mode for path monitoring")
 
 }
 
@@ -144,31 +147,84 @@ func runObserve(cmd *cobra.Command, args []string) {
 
 		// SCHEDULERS
 		schedule := getScheduleForPolicy(policy, config.Flags.PolicySchedule)
-		if schedule == "" {
-			log.Warn().Str("policy", policy.ID).Msg("No schedule found for policy, skipping")
+		if schedule == "" && policy.Observe == "" && policy.Runtime.Observe == "" {
+			log.Warn().Str("policy", policy.ID).Msg("No schedule available for policy, skipping")
 			continue
 		}
 
-		if !validateCronExpression(schedule) {
+		if schedule != "" && !validateCronExpression(schedule) {
 			log.Error().Str("policy", policy.ID).Str("schedule", schedule).Msg("Invalid cron expression, skipping")
 			continue
 		}
 		if policy.Type != "api" && policy.Type != "runtime" && policy.Type != "rego" {
 			policy.Metadata.TargetInfo = preparePolicyPaths(policy, allFileInfos)
 		}
 
-		run = true
+		if schedule != "" {
+			run = true
 
-		policyTask := createPolicyTask(policy, dispatcher)
-		taskr.Task(schedule, policyTask)
-		log.Info().Str("policy", policy.ID).Str("schedule", schedule).Msg("Added policy to Scheduler")
+			policyTask := createPolicyTask(policy, dispatcher)
+			taskr.Task(schedule, policyTask)
+			log.Info().Str("policy", policy.ID).Str("schedule", schedule).Msg("Added policy to Scheduler")
+		}
+
+		if (policy.Observe != "" && policy.Schedule != "") || (policy.Runtime.Observe != "" && policy.Schedule != "") {
+			log.Error().Str("policy", policy.ID).Msg("Policy with both SCHEDULE and OBSERVE defined. Skipping OBSERVE directive")
+			continue
+		}
+
+		if policy.Type != "runtime" && policy.Observe != "" {
+
+			exists, isDirectory, _ := PathInfo(policy.Observe)
+
+			if exists && !PolicyExistsInCache(policy.Observe) {
+
+				overlaps, overlapWith := detectOverlap(observeList, policy.Observe)
+
+				if overlaps {
+					log.Error().Str("policy", policy.ID).Msgf("Observe path overlaps with another policy at : %s", overlapWith)
+					continue
+				}
+
+				observeList = append(observeList, policy.Observe)
+
+				log.Debug().Str("policy", policy.ID).Bool("exists", exists).Bool("isDirectory", isDirectory).Msgf("Setting up watch : %s", policy.Observe)
+
+				StorePolicyInCache(policy.Observe, policy)
+
+				log.Debug().Int("Cache count", GetPolicyCacheCount()).Msg("Cache Status")
+
+				if PolicyExistsInCache(policy.Observe) {
+					log.Info().Str("policy", policy.ID).Str("Observe", policy.Observe).Msg("Added policy to Path Watcher")
+
+					run = true
+
+					//path watcher
+					go func() {
+						defer func() {
+							if r := recover(); r != nil {
+								log.Error().Interface("recover", r).Msg("Panic in path watcher goroutine")
+							}
+						}()
+						watchPaths(policy.Observe)
+					}()
+
+				} else {
+					log.Warn().Str("policy", policy.ID).Msg("Failed Caching the policy - investigate")
+				}
+
+			} else {
+				log.Warn().Str("policy", policy.ID).Str("path", policy.Observe).Msg("Runtime observe has invalid path, skipping")
+			}
+
+		}
 
 		//PATH WATCHERS
 		if policy.Type == "runtime" && policy.Runtime.Observe != "" {
 
 			exists, isDirectory, _ := PathInfo(policy.Runtime.Observe)
 
-			if exists {
+			if exists && !PolicyExistsInCache(policy.Runtime.Observe) {
 
 				log.Debug().Str("policy", policy.ID).Bool("exists", exists).Bool("isDirectory", isDirectory).Msgf("Setting up watch : %s", policy.Runtime.Observe)
 
@@ -179,6 +235,8 @@ func runObserve(cmd *cobra.Command, args []string) {
 				if PolicyExistsInCache(policy.Runtime.Observe) {
 					log.Info().Str("policy", policy.ID).Str("Observe", policy.Runtime.Observe).Msg("Added policy to Path Watcher")
 
+					run = true
+
 					//path watcher
 					go func() {
 						defer func() {
@@ -194,7 +252,7 @@ func runObserve(cmd *cobra.Command, args []string) {
 				}
 
 			} else {
-				log.Warn().Str("policy", policy.ID).Str("path", policy.Runtime.Observe).Msg("Runtime observe has invalid path, skipping")
+				log.Error().Str("policy", policy.ID).Str("path", policy.Runtime.Observe).Msg("Runtime observe has invalid path, skipping")
 			}
 
 		}

cmd/policy.go

@@ -68,6 +68,7 @@ type Policy struct {
 	Enforcement []Enforcement `yaml:"enforcement"`
 	Metadata    Metadata      `yaml:"metadata"`
 	FilePattern string        `yaml:"filepattern"`
+	Observe     string        `yaml:"observe"`
 	Schema      Schema        `yaml:"_schema"`
 	Rego        Rego          `yaml:"_rego"`
 	Regex       []string      `yaml:"_regex"`

cmd/root.go

@@ -123,6 +123,10 @@ func setupLogging() {
 		log.Info().Msgf("Host Data: %s", hostData)
 		log.Info().Msgf("Host Fingerprint: %s", hostFingerprint)
 
+		// ----------------------------------------------
+		// ---------------------------------------------- EXPERIMENTAL END
+		// ----------------------------------------------
+
 	}
 	if silentMode {
 

cmd/target.go

@@ -151,3 +151,25 @@ func processIgnorePatterns(ignorePatterns []string) []string {
 	}
 	return processedPatterns
 }
+
+func detectOverlap(paths []string, targetPath string) (bool, string) {
+	// Normalize the target path
+	targetPath = filepath.Clean(targetPath)
+
+	for _, path := range paths {
+		// Normalize the current path
+		path = filepath.Clean(path)
+
+		// Check if the target path is the same as or a subdirectory of the current path
+		if targetPath == path || strings.HasPrefix(targetPath, path+string(filepath.Separator)) {
+			return true, path
+		}
+
+		// Check if the current path is a subdirectory of the target path
+		if strings.HasPrefix(path, targetPath+string(filepath.Separator)) {
+			return true, targetPath
+		}
+	}
+
+	return false, ""
+}

cmd/watch.go

@@ -8,12 +8,20 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/segmentio/ksuid"
 )
 
+const eventCacheDuration = 2000 * time.Millisecond // Configurable duration
+
+type cachedEvent struct {
+	event fsnotify.Event
+	timer *time.Timer
+}
+
 type HostInfo struct {
 	Hostname     string
 	OS           string
@@ -35,7 +43,16 @@ func watchPaths(paths ...string) {
 	defer w.Close()
 
 	// Start listening for events.
-	go watchLoop(w, paths)
+	switch observeMode {
+	case "last":
+		go watchLoopLastEvent(w, paths)
+	case "first":
+		go watchLoopFirstEvent(w, paths)
+	case "all":
+		go watchLoopAllEvents(w, paths)
+	default:
+		go watchLoopFirstEvent(w, paths)
+	}
 
 	// Add all paths from the commandline.
 	for _, p := range paths {
@@ -50,7 +67,7 @@ func watchPaths(paths ...string) {
 	<-make(chan struct{}) // Block forever
 }
 
-func watchLoop(w *fsnotify.Watcher, watchedPaths []string) {
+func watchLoopAllEvents(w *fsnotify.Watcher, watchedPaths []string) {
 	for {
 		select {
 		case err, ok := <-w.Errors:
@@ -88,11 +105,143 @@ func watchLoop(w *fsnotify.Watcher, watchedPaths []string) {
 	}
 }
 
+func watchLoopFirstEvent(w *fsnotify.Watcher, watchedPaths []string) {
+	eventCache := make(map[string]time.Time)
+	var cacheMutex sync.Mutex
+
+	for {
+		select {
+		case err, ok := <-w.Errors:
+			if !ok {
+				return
+			}
+			log.Error().Msgf("Watcher error: %s", err)
+
+		case event, ok := <-w.Events:
+			if !ok {
+				return
+			}
+
+			log.Debug().Msgf("Watcher caught [%s] on [%s]", event.Op.String(), event.Name)
+
+			// Check if we should process this event
+			cacheMutex.Lock()
+			lastEventTime, exists := eventCache[event.Name]
+			now := time.Now()
+			if !exists || now.Sub(lastEventTime) > eventCacheDuration {
+				// Process the event and update the cache
+				eventCache[event.Name] = now
+				cacheMutex.Unlock()
+
+				// Process the event in a goroutine to avoid blocking
+				go processEvent(event)
+
+				// Clean up old cache entries
+				go cleanEventCache(&eventCache, &cacheMutex)
+			} else {
+				cacheMutex.Unlock()
+				log.Debug().Msgf("Ignored duplicate event for [%s] within cache duration", event.Name)
+			}
+
+			// Re-add the watch for the file if it was removed or renamed
+			if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
+				for _, path := range watchedPaths {
+					if path == event.Name {
+						// Wait a short time for the file to be recreated/renamed
+						time.Sleep(100 * time.Millisecond)
+						if err := w.Add(path); err != nil {
+							log.Error().Msgf("Failed to re-add watch for %s: %s", path, err)
+						} else {
+							log.Debug().Msgf("Re-added watch for %s", path)
+						}
+						break
+					}
+				}
+			}
+		}
+	}
+}
+
+func watchLoopLastEvent(w *fsnotify.Watcher, watchedPaths []string) {
+	eventCache := make(map[string]*cachedEvent)
+	var cacheMutex sync.Mutex
+
+	for {
+		select {
+		case err, ok := <-w.Errors:
+			if !ok {
+				return
+			}
+			log.Error().Msgf("Watcher error: %s", err)
+
+		case event, ok := <-w.Events:
+			if !ok {
+				return
+			}
+
+			log.Debug().Msgf("Watcher caught [%s] on [%s]", event.Op.String(), event.Name)
+
+			cacheMutex.Lock()
+			if cached, exists := eventCache[event.Name]; exists {
+				// Stop the existing timer
+				cached.timer.Stop()
+				// Update the cached event
+				cached.event = event
+				// Reset the timer
+				cached.timer.Reset(eventCacheDuration)
+			} else {
+				// Create a new timer for this event
+				timer := time.AfterFunc(eventCacheDuration, func() {
+					processLastEvent(event.Name, &eventCache, &cacheMutex)
+				})
+				eventCache[event.Name] = &cachedEvent{event: event, timer: timer}
+			}
+			cacheMutex.Unlock()
+
+			// Re-add the watch for the file if it was removed or renamed
+			if event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename) {
+				for _, path := range watchedPaths {
+					if path == event.Name {
+						// Wait a short time for the file to be recreated/renamed
+						time.Sleep(100 * time.Millisecond)
+						if err := w.Add(path); err != nil {
+							log.Error().Msgf("Failed to re-add watch for %s: %s", path, err)
+						} else {
+							log.Debug().Msgf("Re-added watch for %s", path)
+						}
+						break
+					}
+				}
+			}
+		}
+	}
+}
+
+// For a configurable interval:
+// var eventCacheDuration time.Duration = 1000 * time.Millisecond
+// func SetEventCacheDuration(milliseconds int) {
+// 	eventCacheDuration = time.Duration(milliseconds) * time.Millisecond
+// }
+
+func processLastEvent(path string, cache *map[string]*cachedEvent, mutex *sync.Mutex) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	if cached, exists := (*cache)[path]; exists {
+		// Process the most recent event
+		processEvent(cached.event)
+		// Remove the event from the cache
+		delete(*cache, path)
+	}
+}
+
 func processEvent(e fsnotify.Event) {
 	log.Debug().Str("fs", e.Name).Msg(e.String())
 
 	policy, ok := LoadPolicyFromCache(e.Name)
 
+	log.Info().Msgf("Processing event [%s] on [%s]", e.Op.String(), e.Name)
+
 	// Check if the watcher is targeting the directory
 	if !ok {
 		directoryCheck := GetDirectory(e.Name)
@@ -178,3 +327,15 @@ func FingerprintHost(hostInfo *HostInfo) (string, string, error) {
 	fingerprint := hex.EncodeToString(hash.Sum(nil))
 	return data, fingerprint, nil
 }
+
+func cleanEventCache(cache *map[string]time.Time, mutex *sync.Mutex) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	now := time.Now()
+	for path, lastEventTime := range *cache {
+		if now.Sub(lastEventTime) > eventCacheDuration {
+			delete(*cache, path)
+		}
+	}
+}

cmd/worker.go

@@ -22,6 +22,8 @@ func processPolicyInWorker(e event.Event, policyType string) error {
 	targetDir, _ := e.Get("targetDir").(string)
 	filePaths, _ := e.Get("filePaths").([]string)
 
+	log.Debug().Str("policy", policy.ID).Msgf("Working [%s] [%s]", targetDir, filePaths)
+
 	switch policyType {
 	case "scan":
 		return ProcessScanType(policy, rgPath, targetDir, filePaths)