Specify multiple http urls to probe

src/in-binary.c

@@ -325,6 +325,7 @@ parse_banner6(struct Output *out, unsigned char *buf, size_t length,
                 record.port,
                 record.app_proto,   /* HTTP, SSL, SNMP, etc. */
                 record.ttl, /* ttl */
+                NULL, 0,
                 buf+offset, (unsigned)(length-offset)
                 );
 }
@@ -364,6 +365,7 @@ parse_banner3(struct Output *out, unsigned char *buf, size_t buf_length)
                 record.port,
                 record.app_proto,
                 0, /* ttl */
+                NULL, 0,
                 buf+12, (unsigned)buf_length-12
                 );
 }
@@ -404,6 +406,7 @@ parse_banner4(struct Output *out, unsigned char *buf, size_t buf_length)
                 record.port,
                 record.app_proto,   /* HTTP, SSL, SNMP, etc. */
                 0, /* ttl */
+                NULL, 0,
                 buf+13, (unsigned)buf_length-13
                 );
 }
@@ -462,6 +465,7 @@ parse_banner9(struct Output *out, unsigned char *buf, size_t buf_length,
                 record.port,
                 record.app_proto,   /* HTTP, SSL, SNMP, etc. */
                 record.ttl, /* ttl */
+                NULL, 0,
                 data, (unsigned)data_length
                 );
 }

src/main-conf.c

@@ -2345,6 +2345,19 @@ static int SET_debug_tcp(struct Masscan *masscan, const char *name, const char *
 }
 
 
+static int SET_output_probes(struct Masscan *masscan, const char *name, const char *value)
+{
+    UNUSEDPARM(name);
+    if (masscan->echo) {
+        if (masscan->is_output_probes || masscan->echo_all)
+            fprintf(masscan->echo, "output-probes = %s\n", masscan->is_output_probes?"true":"false");
+       return 0;
+    }
+    masscan->is_output_probes = parseBoolean(value);
+    return CONF_OK;
+}
+
+
 
 struct ConfigParameter {
     const char *name;
@@ -2395,6 +2408,7 @@ struct ConfigParameter config_parameters[] = {
     {"output-noshow",   SET_output_noshow,      0,      {"noshow",0}},
     {"output-show-open",SET_output_show_open,   F_BOOL, {"open", "open-only", 0}},
     {"output-append",   SET_output_append,      0,      {"append-output",0}},
+    {"output-probes",   SET_output_probes,      F_BOOL, {0}},
     {"rotate",          SET_rotate_time,        0,      {"output-rotate", "rotate-output", "rotate-time", 0}},
     {"rotate-dir",      SET_rotate_directory,   0,      {"output-rotate-dir", "rotate-directory", 0}},
     {"rotate-offset",   SET_rotate_offset,      0,      {"output-rotate-offset", 0}},

src/masscan.h

@@ -206,6 +206,7 @@ struct Masscan
     unsigned is_hello_http:1;    /* --hello=http, use HTTP on all ports */
     unsigned is_scripting:1;    /* whether scripting is needed */
     unsigned is_capture_servername:1; /* --capture servername */
+    unsigned is_output_probes:1; /* --output-probes */
 
     /** Packet template options, such as whether we should add a TCP MSS
      * value, or remove it from the packet */

src/out-binary.c

@@ -217,6 +217,7 @@ static void
 binary_out_banner_ipv6(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     unsigned char foo[32768];
@@ -294,6 +295,7 @@ static void
 binary_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     unsigned char foo[32768];
@@ -302,7 +304,7 @@ binary_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     static const unsigned HeaderLength = 14;
 
     if (ip.version == 6) {
-        binary_out_banner_ipv6(out, fp, timestamp, ip, ip_proto, port, proto, ttl, px, length);
+        binary_out_banner_ipv6(out, fp, timestamp, ip, ip_proto, port, proto, ttl, probe, probe_length, px, length);
         return;
     }
 

src/out-certs.c

@@ -51,6 +51,7 @@ cert_out_banner(struct Output *out, FILE *fp, time_t timestamp,
                 ipaddress ip, unsigned ip_proto, unsigned port,
                 enum ApplicationProtocol proto, 
                 unsigned ttl,
+                const unsigned char *probe, unsigned probe_length,
                 const unsigned char *px, unsigned length)
 {
     unsigned i;
@@ -63,6 +64,8 @@ cert_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     UNUSEDPARM(ttl);
     UNUSEDPARM(proto);
     UNUSEDPARM(port);
+    UNUSEDPARM(probe);
+    UNUSEDPARM(probe_length);
 
     if (length > 5 && memcmp(px, "cert:", 5) == 0) {
         px += 5;

src/out-grepable.c

@@ -179,9 +179,11 @@ static void
 grepable_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     char banner_buffer[MAX_BANNER_LENGTH];
+    char probe_buffer[MAX_PROBE_LENGTH];
     ipaddress_formatted_t fmt;
 
     UNUSEDPARM(ttl);
@@ -195,6 +197,12 @@ grepable_out_banner(struct Output *out, FILE *fp, time_t timestamp,
 
     fprintf(fp, "\tService: %s", masscan_app_to_string(proto));
 
+    if (out->masscan->is_output_probes) {
+        normalize_string(probe, probe_length, probe_buffer, sizeof(probe_buffer));
+
+        fprintf(fp, "\tProbe: %s\n", probe_buffer);
+    }
+
     normalize_string(px, length, banner_buffer, sizeof(banner_buffer));
 
     fprintf(fp, "\tBanner: %s\n", banner_buffer);

src/out-hostonly.c

@@ -46,6 +46,7 @@ static void
 hostonly_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 { /* SYN only - no banner */
     ipaddress_formatted_t fmt = ipaddress_fmt(ip);
@@ -57,6 +58,8 @@ hostonly_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     UNUSEDPARM(ip);
     UNUSEDPARM(ip_proto);
     UNUSEDPARM(proto);
+    UNUSEDPARM(probe);
+    UNUSEDPARM(probe_length);
     UNUSEDPARM(px);
     UNUSEDPARM(length);
     fprintf(fp, "%s\n", fmt.string);

src/out-json.c

@@ -1,4 +1,5 @@
 #include "output.h"
+#include "masscan.h"
 #include "masscan-app.h"
 #include "masscan-status.h"
 #include "util-safefunc.h"
@@ -103,6 +104,7 @@ json_out_banner(struct Output *out, FILE *fp, time_t timestamp,
                ipaddress ip, unsigned ip_proto, unsigned port,
                enum ApplicationProtocol proto,
                unsigned ttl,
+               const unsigned char *probe, unsigned probe_length,
                const unsigned char *px, unsigned length)
 {
     char banner_buffer[65536];
@@ -121,13 +123,23 @@ json_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     fprintf(fp, "{ ");
     fmt = ipaddress_fmt(ip);
     fprintf(fp, "  \"ip\": \"%s\", ", fmt.string);
-    fprintf(fp, "  \"timestamp\": \"%d\", \"ports\": [ {\"port\": %u, \"proto\": \"%s\", \"service\": {\"name\": \"%s\", \"banner\": \"%s\"} } ] ",
-            (int) timestamp,
-            port,
-            name_from_ip_proto(ip_proto),
-            masscan_app_to_string(proto),
-            normalize_json_string(px, length, banner_buffer, sizeof(banner_buffer))
-            );
+    if (out->masscan->is_output_probes)
+        fprintf(fp, "  \"timestamp\": \"%d\", \"ports\": [ {\"port\": %u, \"proto\": \"%s\", \"service\": {\"name\": \"%s\", \"probe\": \"%s\", \"banner\": \"%s\"} } ] ",
+                (int) timestamp,
+                port,
+                name_from_ip_proto(ip_proto),
+                masscan_app_to_string(proto),
+                normalize_json_string(probe, probe_length, banner_buffer, sizeof(banner_buffer)),
+                normalize_json_string(px, length, banner_buffer, sizeof(banner_buffer))
+                );
+    else
+        fprintf(fp, "  \"timestamp\": \"%d\", \"ports\": [ {\"port\": %u, \"proto\": \"%s\", \"service\": {\"name\": \"%s\", \"banner\": \"%s\"} } ] ",
+                (int) timestamp,
+                port,
+                name_from_ip_proto(ip_proto),
+                masscan_app_to_string(proto),
+                normalize_json_string(px, length, banner_buffer, sizeof(banner_buffer))
+                );
     fprintf(fp, "}\n");
 
     UNUSEDPARM(out);

src/out-ndjson.c

@@ -1,4 +1,5 @@
 #include "output.h"
+#include "masscan.h"
 #include "masscan-app.h"
 #include "masscan-status.h"
 #include "util-safefunc.h"
@@ -99,6 +100,7 @@ ndjson_out_banner(struct Output *out, FILE *fp, time_t timestamp,
                  ipaddress ip, unsigned ip_proto, unsigned port,
                  enum ApplicationProtocol proto,
                  unsigned ttl,
+                 const unsigned char *probe, unsigned probe_length,
                  const unsigned char *px, unsigned length)
 {
     char banner_buffer[65536];
@@ -110,13 +112,23 @@ ndjson_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     fprintf(fp, "{");
     fmt = ipaddress_fmt(ip);
     fprintf(fp, "\"ip\":\"%s\",", fmt.string);
-    fprintf(fp, "\"timestamp\":\"%d\",\"port\":%u,\"proto\":\"%s\",\"rec_type\":\"banner\",\"data\":{\"service_name\":\"%s\",\"banner\":\"%s\"}",
-            (int) timestamp,
-            port,
-            name_from_ip_proto(ip_proto),
-            masscan_app_to_string(proto),
-            normalize_ndjson_string(px, length, banner_buffer, sizeof(banner_buffer))
-            );
+    if (out->masscan->is_output_probes)
+        fprintf(fp, "\"timestamp\":\"%d\",\"port\":%u,\"proto\":\"%s\",\"rec_type\":\"banner\",\"data\":{\"service_name\":\"%s\", \"probe\": \"%s\", \"banner\":\"%s\"}",
+                (int) timestamp,
+                port,
+                name_from_ip_proto(ip_proto),
+                masscan_app_to_string(proto),
+                normalize_ndjson_string(probe, probe_length, banner_buffer, sizeof(banner_buffer)),
+                normalize_ndjson_string(px, length, banner_buffer, sizeof(banner_buffer))
+                );
+    else
+        fprintf(fp, "\"timestamp\":\"%d\",\"port\":%u,\"proto\":\"%s\",\"rec_type\":\"banner\",\"data\":{\"service_name\":\"%s\", \"banner\":\"%s\"}",
+                (int) timestamp,
+                port,
+                name_from_ip_proto(ip_proto),
+                masscan_app_to_string(proto),
+                normalize_ndjson_string(px, length, banner_buffer, sizeof(banner_buffer))
+                );
     // fprintf(fp, "\"timestamp\":\"%d\",\"ports\":[{\"port\":%u,\"proto\":\"%s\",\"service\":{\"name\":\"%s\",\"banner\":\"%s\"}}]",
     //         (int) timestamp,
     //         port,

src/out-null.c

@@ -55,6 +55,7 @@ static void
 null_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     UNUSEDPARM(ttl);
@@ -65,6 +66,8 @@ null_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     UNUSEDPARM(ip_proto);
     UNUSEDPARM(port);
     UNUSEDPARM(proto);
+    UNUSEDPARM(probe);
+    UNUSEDPARM(probe_length);
     UNUSEDPARM(px);
     UNUSEDPARM(length);
 

src/out-redis.c

@@ -321,6 +321,7 @@ static void
 redis_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     UNUSEDPARM(ttl);
@@ -331,6 +332,8 @@ redis_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     UNUSEDPARM(ip_proto);
     UNUSEDPARM(port);
     UNUSEDPARM(proto);
+    UNUSEDPARM(probe);
+    UNUSEDPARM(probe_length);
     UNUSEDPARM(px);
     UNUSEDPARM(length);
 

src/out-text.c

@@ -52,24 +52,39 @@ static void
 text_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 {
     char banner_buffer[MAX_BANNER_LENGTH];
+    char probe_buffer[MAX_PROBE_LENGTH];
     ipaddress_formatted_t fmt = ipaddress_fmt(ip);
 
 
     UNUSEDPARM(out);
     UNUSEDPARM(ttl);
 
-    fprintf(fp, "%s %s %u %s %u %s %s\n",
-        "banner",
-        name_from_ip_proto(ip_proto),
-        port,
-        fmt.string,
-        (unsigned)timestamp,
-        masscan_app_to_string(proto),
-        normalize_string(px, length, banner_buffer, sizeof(banner_buffer))
-        );
+    if (out->masscan->is_output_probes)
+        fprintf(fp, "%s %s %u %s %u %s %s %s\n",
+            "banner",
+            name_from_ip_proto(ip_proto),
+            port,
+            fmt.string,
+            (unsigned)timestamp,
+            masscan_app_to_string(proto),
+            normalize_string(probe, probe_length, probe_buffer, sizeof(probe_buffer)),
+            normalize_string(px, length, banner_buffer, sizeof(banner_buffer))
+            );
+    else
+        fprintf(fp, "%s %s %u %s %u %s %s\n",
+            "banner",
+            name_from_ip_proto(ip_proto),
+            port,
+            fmt.string,
+            (unsigned)timestamp,
+            masscan_app_to_string(proto),
+            normalize_string(px, length, banner_buffer, sizeof(banner_buffer))
+            );
+
 }
 
 

src/out-unicornscan.c

@@ -62,6 +62,7 @@ static void
 unicornscan_out_banner(struct Output *out, FILE *fp, time_t timestamp,
         ipaddress ip, unsigned ip_proto, unsigned port,
         enum ApplicationProtocol proto, unsigned ttl,
+        const unsigned char *probe, unsigned probe_length,
         const unsigned char *px, unsigned length)
 { /* SYN only - no banner */
     UNUSEDPARM(out);
@@ -72,6 +73,8 @@ unicornscan_out_banner(struct Output *out, FILE *fp, time_t timestamp,
     UNUSEDPARM(ip);
     UNUSEDPARM(ip_proto);
     UNUSEDPARM(proto);
+    UNUSEDPARM(probe);
+    UNUSEDPARM(probe_length);
     UNUSEDPARM(px);
     UNUSEDPARM(length);