Improve information about regexes (#22)

* Improve descriptions in Options tab - Added sections to regexes created with the new button - Improved description labels - License header for files * Fix sections of regexes imported from CSV - Set the sections for regexes added through CSV import * docs: New "About the used regexes" section
CYS4srl · Mar 28, 2023 · 81b693f · 81b693f
1 parent 9952628
commit 81b693f
Show file tree

Hide file tree

Showing 7 changed files with 81 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@
   * [Introduction](#introduction)
     * [Features](#features)
     * [Screenshots](#screenshots)
+    * [About the used regexes](#about-the-used-regexes)
   * [Installation](#installation)
     * [Using the BApp Store](#using-the-bapp-store)
     * [Manual install](#manual-install)
@@ -44,6 +45,24 @@ Options tab to configure filters and scanner options:
 
 ![Options tab](images/tab-options.png)
 
+### About the used regexes
+
+We aim to provide a default set of regexes that can be used in as many cases as possible without numerous false positives.
+
+As the source, many regexes are written by us, and any other should have the appropriate mention in the [References](#references) section.
+
+Each Proxy list row is divided into sections to improve the matching results and reduce the scan times. As of now, there are five sections:
+
+- Request
+  - Request URL
+  - Request Headers
+  - Request Body
+- Response
+  - Response Headers
+  - Response Body
+
+The extension works with two lists of regexes. One list is for general regexes, which only matches within the Response sections; The other is for filename extensions and only matches the Request URL.
+
 ## Installation
 
 ### Using the BApp Store
@@ -97,7 +116,7 @@ The BApp can be compiled with Maven by following these steps:
 
 1. View > Tool Windows > Maven.
 2. On the new right panel expand the Lifecycle folder.
-3. Double-click on install.
+3. Double-click on "Install".
 
 The compiled extension will be in the "/target" folder.
 
@@ -117,4 +136,7 @@ Check out [our site](https://cys4.com/) and [our blog](https://blog.cys4.com/) f
 
 ## References
 
-- [shhgit](https://github.com/eth0izzle/shhgit/blob/master/config.yaml): Regexes and File Extensions lists used in this project.
+The following is a list of sources for some regexes used in this extension. Many thanks to all!
+
+- https://github.com/eth0izzle/shhgit
+- https://github.com/streaak/keyhacks
diff --git a/src/main/java/com/cys4/sensitivediscoverer/controller/Utils.java b/src/main/java/com/cys4/sensitivediscoverer/controller/Utils.java
@@ -4,7 +4,9 @@
 */
 package com.cys4.sensitivediscoverer.controller;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 import java.util.stream.Collectors;

diff --git a/src/main/java/com/cys4/sensitivediscoverer/model/JsonRegexEntity.java b/src/main/java/com/cys4/sensitivediscoverer/model/JsonRegexEntity.java
@@ -1,3 +1,7 @@
+/*
+Copyright (C) 2023 CYS4 Srl
+See the file 'LICENSE' for copying permission
+*/
 package com.cys4.sensitivediscoverer.model;
 
 import java.util.List;

diff --git a/src/main/java/com/cys4/sensitivediscoverer/model/ProxyItemSection.java b/src/main/java/com/cys4/sensitivediscoverer/model/ProxyItemSection.java
@@ -1,3 +1,7 @@
+/*
+Copyright (C) 2023 CYS4 Srl
+See the file 'LICENSE' for copying permission
+*/
 package com.cys4.sensitivediscoverer.model;
 
 import java.util.EnumSet;
@@ -47,4 +51,16 @@ public static EnumSet<ProxyItemSection> parseSectionsToMatch(List<String> sectio
             .filter(Objects::nonNull)
             .collect(Collectors.toCollection(() -> EnumSet.noneOf(ProxyItemSection.class)));
     }
+
+    @Override
+    public String toString() {
+        return switch (this) {
+            case REQ_URL -> "RequestURL";
+            case REQ_BODY -> "RequestBody";
+            case REQ_HEADERS -> "RequestHeaders";
+            case RES_BODY -> "ResponseBody";
+            case RES_HEADERS -> "ResponseHeaders";
+            default -> this.name();
+        };
+    }
 }
diff --git a/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java b/src/main/java/com/cys4/sensitivediscoverer/model/RegexEntity.java
@@ -4,7 +4,8 @@
 */
 package com.cys4.sensitivediscoverer.model;
 
-import java.util.*;
+import java.util.EnumSet;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -15,12 +16,6 @@ public class RegexEntity {
     private final String description;
     private final EnumSet<ProxyItemSection> sections;
 
-    /**
-     * Used to import from CSV where there's only the description and the regex
-     * @param description
-     * @param regex
-     * @throws IllegalArgumentException
-     */
     public RegexEntity(String description, String regex) throws IllegalArgumentException {
         this(description, regex, true, ProxyItemSection.getDefault());
     }

diff --git a/src/main/java/com/cys4/sensitivediscoverer/scanner/BurpLeaksScanner.java b/src/main/java/com/cys4/sensitivediscoverer/scanner/BurpLeaksScanner.java
@@ -12,6 +12,7 @@
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
 
+import javax.swing.*;
 import java.lang.reflect.Type;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
@@ -21,8 +22,6 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import javax.swing.JProgressBar;
-
 public class BurpLeaksScanner {
 
     private final MainUI mainUI;

diff --git a/src/main/java/com/cys4/sensitivediscoverer/ui/MainUI.java b/src/main/java/com/cys4/sensitivediscoverer/ui/MainUI.java
@@ -9,6 +9,7 @@
 import burp.ITextEditor;
 import burp.SpringUtilities;
 import com.cys4.sensitivediscoverer.model.LogEntity;
+import com.cys4.sensitivediscoverer.model.ProxyItemSection;
 import com.cys4.sensitivediscoverer.model.RegexEntity;
 import com.cys4.sensitivediscoverer.scanner.BurpLeaksScanner;
 import com.cys4.sensitivediscoverer.seed.RegexSeeder;
@@ -24,8 +25,8 @@
 import java.io.*;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.*;
 import java.util.List;
+import java.util.*;
 import java.util.function.Supplier;
 import java.util.regex.Matcher;
 
@@ -432,20 +433,29 @@ private List<JComponent> createLogger_AnalyzeHTTPHistory(JPanel tabPanelLogger)
     private JPanel createOptionsPanel() {
         JPanel tabPaneOptions = new JPanel();
         tabPaneOptions.setLayout(new BoxLayout(tabPaneOptions, BoxLayout.Y_AXIS));
-        tabPaneOptions.setBorder(BorderFactory.createTitledBorder("Configuration"));
 
         // Configuration
         JPanel configurationsPanel = createOptions_Configurations();
         tabPaneOptions.add(configurationsPanel);
         tabPaneOptions.add(new JSeparator());
 
         // Regex
-        createOptions_Regex(tabPaneOptions, createOptions_Regex_Title(), RegexSeeder::getGeneralRegexes, this.generalRegexList)
+        createOptions_Regex(
+                tabPaneOptions,
+                createOptions_Regex_Title(),
+                RegexSeeder::getGeneralRegexes,
+                this.generalRegexList,
+                ProxyItemSection.getDefault())
             .forEach(tabPaneOptions::add);
         tabPaneOptions.add(new JSeparator());
 
         // Extensions
-        createOptions_Regex(tabPaneOptions, createOptions_Extensions_Title(), RegexSeeder::getExtensionRegexes, this.extensionsRegexList)
+        createOptions_Regex(
+                tabPaneOptions,
+                createOptions_Extensions_Title(),
+                RegexSeeder::getExtensionRegexes,
+                this.extensionsRegexList,
+                EnumSet.of(ProxyItemSection.REQ_URL))
             .forEach(tabPaneOptions::add);
 
         return tabPaneOptions;
@@ -477,14 +487,20 @@ private JPanel createOptions_ParagraphSection(String title, String description)
     }
 
     private JPanel createOptions_Regex_Title() {
-        return createOptions_ParagraphSection("Regex List", "In this section you can manage the regex list.");
+        return createOptions_ParagraphSection("Regex List", "This section contains general regexes that try to match only within the response.");
     }
 
     private JPanel createOptions_Extensions_Title() {
-        return createOptions_ParagraphSection("Extensions List", "In this section you can manage the extension list.");
+        return createOptions_ParagraphSection("Extensions List", "This section contains regexes for filename extensions. These regexes try to match only the URL of the request.");
     }
 
-    private List<JComponent> createOptions_Regex(JPanel tabPaneOptions, JPanel optionsTitlePanel, Supplier<List<RegexEntity>> resetRegexSeeder, List<RegexEntity> regexEntities) {
+    private List<JComponent> createOptions_Regex(
+            JPanel tabPaneOptions,
+            JPanel optionsTitlePanel,
+            Supplier<List<RegexEntity>> resetRegexSeeder,
+            List<RegexEntity> regexEntities,
+            EnumSet<ProxyItemSection> newRegexesSections)
+    {
         var ctx = new Object() {
             final List<RegexEntity> regexList = regexEntities;
         };
@@ -541,7 +557,12 @@ private List<JComponent> createOptions_Regex(JPanel tabPaneOptions, JPanel optio
         btnNewRegex.addActionListener(actionEvent -> {
             String[] labels = {"Regex: ", "Description: "};
             //Create and populate the panel.
+            JPanel mainPanel = new JPanel();
+            mainPanel.setLayout(new BoxLayout(mainPanel, BoxLayout.Y_AXIS));
+            JLabel labelSummary = new JLabel("The new regex will only match: " + newRegexesSections.toString(), JLabel.TRAILING);
+            mainPanel.add(labelSummary);
             JPanel inputPanel = new JPanel(new SpringLayout());
+            mainPanel.add(inputPanel);
             JLabel labelExpression = new JLabel(labels[0], JLabel.TRAILING);
             inputPanel.add(labelExpression);
             JTextField textFieldReg = new JTextField(10);
@@ -557,14 +578,14 @@ private List<JComponent> createOptions_Regex(JPanel tabPaneOptions, JPanel optio
                     labels.length, 2, //rows, cols
                     6, 6,        //initX, initY
                     6, 6);       //xPad, yPad
-            int returnValue = JOptionPane.showConfirmDialog(tabPaneOptions, inputPanel, "Add a regular expression", JOptionPane.YES_NO_OPTION);
+            int returnValue = JOptionPane.showConfirmDialog(tabPaneOptions, mainPanel, "Add a regular expression", JOptionPane.YES_NO_OPTION);
             if (returnValue != JOptionPane.YES_OPTION) return;
 
             String expression = textFieldReg.getText();
             String description = textFieldDesc.getText();
 
             int row = ctx.regexList.size();
-            ctx.regexList.add(new RegexEntity(description, expression));
+            ctx.regexList.add(new RegexEntity(description, expression, true, newRegexesSections));
             modelReg.fireTableRowsInserted(row, row);
 
             tabPaneOptions.validate();
@@ -622,7 +643,7 @@ private List<JComponent> createOptions_Regex(JPanel tabPaneOptions, JPanel optio
                     String description = matcher.group(1);
                     String regex = matcher.group(2);
 
-                    RegexEntity newRegexEntity = new RegexEntity(description, regex);
+                    RegexEntity newRegexEntity = new RegexEntity(description, regex, true, newRegexesSections);
 
                     if (!ctx.regexList.contains(newRegexEntity)) {
                         ctx.regexList.add(newRegexEntity);