express-node-metrics

This package is a platform for collecting metrics of node and express application.

Supports Restify framework from version 1.3.0 and above

This middleware adds start time to the request, and decorate the end method of express with a new one. At the new end method it add 'X-Response-Time' to the response and the response time to the API metrics using metrics.addApiData method In addition it increments a metric which counts the current active incoming requests which can be found under process.run.activeRequests

express_node_metrics.metrics

Include all the method that collects and retrieves the metrics data.

express_node_metrics.metrics.getAll(reset)

Retrieve all the logged metrics by now.

When execute with reset equals reset=true after retrieving the metrics all the logged data will be reset.

Arguments

reset – Boolean that indicates if to reset all the metrics

Returned function arguments

metrics (required) – an Object representing all of the aggregated metrics. It usually build from the sections: process, api metrics, internal metrics, endpoints metrics and custom metrics if exist.

express_node_metrics.metrics.processMetrics(reset)

Retrieve the process metrics. The process metrics include:

Memory usage - using [node v4 process object] (https://nodejs.org/dist/latest-v4.x/docs/api/process.html#process_process_memoryusage) and [memwatch-next package] (https://www.npmjs.com/package/memwatch-next)
CPU usage - using [pidusage package] (https://www.npmjs.com/package/pidusage)
GC - using [gc-stats package] (https://www.npmjs.com/package/gc-stats)
Event loop latency - using [event-loop-stats package] (https://www.npmjs.com/package/event-loop-stats)
Process usage - process uptime [process.uptime()] and number of current active incoming requests

Arguments

reset – Boolean that indicates if to reset the process metrics

Returned function arguments

metrics (required) – an Object representing the process metrics as describe above. examlpe:

{  "run" : {
       "uptime" : 1090,
       "activeRequests" : 1
  },
   "cpu":{  
      "usage":10.220903395976991
   },
   "memory":{  
      "usage":{  
         "rss":107847680,
         "heapTotal":83625840,
         "heapUsed":60391624
      }
   },
   "eventLoop":{  
      "latency":{  
         "min":0,
         "max":23,
         "num":6242,
         "sum":1581
      }
   },
   "gc":{  
      "lastRun" : 0,
      "time":{  
         "meter":{  
            "mean":0.010381926827711163,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.0035681005137041095,
            "5MinuteRate":0.0024489296340649987,
            "15MinuteRate":0.0010025873653551841
         },
         "histogram":{  
            "min":1,
            "max":1,
            "sum":1,
            "variance":null,
            "mean":1,
            "stddev":0,
            "count":1,
            "median":1,
            "p75":1,
            "p95":1,
            "p99":1,
            "p999":1
         }
      },
      "releasedMem":{  
         "meter":{  
            "mean":0.010382177399303055,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.0035681005137041095,
            "5MinuteRate":0.0024489296340649987,
            "15MinuteRate":0.0010025873653551841
         },
         "histogram":{  
            "min":-1606976,
            "max":-1606976,
            "sum":-1606976,
            "variance":null,
            "mean":-1606976,
            "stddev":0,
            "count":1,
            "median":-1606976,
            "p75":-1606976,
            "p95":-1606976,
            "p99":-1606976,
            "p999":-1606976
         }
      }
   }
}

express_node_metrics.metrics.apiMetrics(reset)

Retrieve all the API metrics that was aggregate until the execution point.ß

When execute with reset=true after retrieving the metrics all the logged data will be reset.

Arguments

reset – Boolean that indicates if to reset the API metrics

Returned function arguments

metrics (required) – an Object representing the API metrics that was aggregated by now. examlpe:

{  
   "global":{  
      "all":{  
         "meter":{  
            "mean":0.011279328252622596,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":65,
            "max":65,
            "sum":65,
            "variance":null,
            "mean":65,
            "stddev":0,
            "count":1,
            "median":65,
            "p75":65,
            "p95":65,
            "p99":65,
            "p999":65
         }
      }
   },
   "statuses":{  
      "200":{  
         "meter":{  
            "mean":0.011279345891387335,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":65,
            "max":65,
            "sum":65,
            "variance":null,
            "mean":65,
            "stddev":0,
            "count":1,
            "median":65,
            "p75":65,
            "p95":65,
            "p99":65,
            "p999":65
         }
      },
      "201":{  
         "meter":{  
            "mean":0.0126722535655588,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004979685410881516,
            "5MinuteRate":0.0026177566530753717,
            "15MinuteRate":0.001025116480975725
         },
         "histogram":{  
            "min":83,
            "max":83,
            "sum":83,
            "variance":null,
            "mean":83,
            "stddev":0,
            "count":1,
            "median":83,
            "p75":83,
            "p95":83,
            "p99":83,
            "p999":83
         }
      },
      "400":{  
         "meter":{  
            "mean":0.011391758628706297,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":6,
            "max":6,
            "sum":6,
            "variance":null,
            "mean":6,
            "stddev":0,
            "count":1,
            "median":6,
            "p75":6,
            "p95":6,
            "p99":6,
            "p999":6
         }
      }
   },
   "methods":{  
      "POST":{  
         "meter":{  
            "mean":0.011279259654897822,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":65,
            "max":65,
            "sum":65,
            "variance":null,
            "mean":65,
            "stddev":0,
            "count":1,
            "median":65,
            "p75":65,
            "p95":65,
            "p99":65,
            "p999":65
         }
      },
      "GET":{  
         "meter":{  
            "mean":0.014505734939935762,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.005882803212940022,
            "5MinuteRate":0.0027064858119822763,
            "15MinuteRate":0.0010365701779111729
         },
         "histogram":{  
            "min":17,
            "max":17,
            "sum":17,
            "variance":null,
            "mean":17,
            "stddev":0,
            "count":1,
            "median":17,
            "p75":17,
            "p95":17,
            "p99":17,
            "p999":17
         }
      },
      "PATCH":{  
         "meter":{  
            "mean":0.03726150471483867,
            "count":1,
            "currentRate":0.037261503188937346,
            "1MinuteRate":0.011458136074669108,
            "5MinuteRate":0.0030925140804589166,
            "15MinuteRate":0.001083679073650417
         },
         "histogram":{  
            "min":13,
            "max":13,
            "sum":13,
            "variance":null,
            "mean":13,
            "stddev":0,
            "count":1,
            "median":13,
            "p75":13,
            "p95":13,
            "p99":13,
            "p999":13
         }
      }
   },
   "endpoints":{  
      "/v1/applications/authenticate|post":{  
         "meter":{  
            "mean":0.01127925238596365,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":65,
            "max":65,
            "sum":65,
            "variance":null,
            "mean":65,
            "stddev":0,
            "count":1,
            "median":65,
            "p75":65,
            "p95":65,
            "p99":65,
            "p999":65
         }
      },
      "/authenticate|post":{  
         "meter":{  
            "mean":0.011360091853355533,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":22,
            "max":22,
            "sum":22,
            "variance":null,
            "mean":22,
            "stddev":0,
            "count":1,
            "median":22,
            "p75":22,
            "p95":22,
            "p99":22,
            "p999":22
         }
      },
      "/v1/applications/|post":{  
         "meter":{  
            "mean":0.012672079014021197,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004979685410881516,
            "5MinuteRate":0.0026177566530753717,
            "15MinuteRate":0.001025116480975725
         },
         "histogram":{  
            "min":83,
            "max":83,
            "sum":83,
            "variance":null,
            "mean":83,
            "stddev":0,
            "count":1,
            "median":83,
            "p75":83,
            "p95":83,
            "p99":83,
            "p999":83
         }
      }
   }
}

express_node_metrics.metrics.internalMetrics(reset)

Retrieve all the internal metrics that was aggregate until the execution point.

When execute with reset=true after retrieving the metrics all the logged data will be reset.

Arguments

reset – Boolean that indicates if to reset the internal metrics

Returned function arguments

metrics (required) – a Number representing the total number of pages for the given query executed on the page. Example:

{  
   "global":{  
      "all":{  
         "meter":{  
            "mean":0.011275864297885328,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":38,
            "max":38,
            "sum":38,
            "variance":null,
            "mean":38,
            "stddev":0,
            "count":1,
            "median":38,
            "p75":38,
            "p95":38,
            "p99":38,
            "p999":38
         }
      }
   },
   "statuses":{  
      "success":{  
         "meter":{  
            "mean":0.011275889368290304,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":38,
            "max":38,
            "sum":38,
            "variance":null,
            "mean":38,
            "stddev":0,
            "count":1,
            "median":38,
            "p75":38,
            "p95":38,
            "p99":38,
            "p999":38
         }
      }
   },
   "methods":{  
      "auditRequest":{  
         "meter":{  
            "mean":0.011275899122149878,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":38,
            "max":38,
            "sum":38,
            "variance":null,
            "mean":38,
            "stddev":0,
            "count":1,
            "median":38,
            "p75":38,
            "p95":38,
            "p99":38,
            "p999":38
         }
      },
      "auditResponse":{  
         "meter":{  
            "mean":0.011280750820217424,
            "count":1,
            "currentRate":0,
            "1MinuteRate":0.004215212696015611,
            "5MinuteRate":0.002531936381998388,
            "15MinuteRate":0.001013789342932559
         },
         "histogram":{  
            "min":9,
            "max":9,
            "sum":9,
            "variance":null,
            "mean":9,
            "stddev":0,
            "count":1,
            "median":9,
            "p75":9,
            "p95":9,
            "p99":9,
            "p999":9
         }
      }
   }
}

express_node_metrics.metrics.addApiData(info)

Aggregate date using the info object in four different sections under apiMetrics:

global - all the northbound API response times aggregated in one place.
statuses - The northbound API response time aggregate by http statuses (info.status - implemented in the middleware using res.statusCode);
methods - The northbound API response time aggregate by http methods (info.method - implemented in the middleware using req.method)
endpoints - The northbound API response time aggregate by route (info.route - implemented in the middleware using req.baseUrl + req.route.path)

each and every one of the sections is [Timer metric] (https://www.npmjs.com/package/measured#timers)

And additionally aggregate the northbound API response time as endpoints metrics in two different sections under endpoints:

<route>.<method>.<status code> - The northbound API response time aggregate by route, method and status code
<route>.<method> - last response time for this endpoint

Arguments

info (required) – the info object that include all the information that needed to aggregate the data.
- status - The http status code of the response.
- method - The http method of the request.
- route - Full express route pattern.
- time - Response time of the specific request.

express_node_metrics.metrics.endPointMetrics(reset)

Retrieve all the endpoint metrics that was aggregate until the execution point.

When execute with reset=true after retrieving the metrics all the logged data will be reset.

Arguments

reset – Boolean that indicates if to reset the internal metrics

Returned function arguments

metrics (required) – a Number representing the total number of pages for the given query executed on the page. Example:

{
  "\/v1\/applications\/testApp|post" : {
    "200" : {
      "meter" : {
        "mean" : 548.6962422190779,
        "15MinuteRate" : 0,
        "1MinuteRate" : 0,
        "count" : 2,
        "currentRate" : 549.281991325061,
        "5MinuteRate" : 0
      },
      "histogram" : {
        "mean" : 30,
        "p75" : 50,
        "count" : 2,
        "median" : 30,
        "p95" : 50,
        "max" : 50,
        "p999" : 50,
        "variance" : 800,
        "stddev" : 28.2842712474619,
        "p99" : 50,
        "sum" : 60,
        "min" : 10
      }
    },
    "lastResponseTime" : 50
  },
  "\/v1\/applications\/testApp|get" : {
    "200" : {
      "meter" : {
        "mean" : 60.44634944811282,
        "15MinuteRate" : 0.0009751209602447366,
        "1MinuteRate" : 0.002352236831272823,
        "count" : 4,
        "currentRate" : 60.44472174128836,
        "5MinuteRate" : 0.002253124031641734
      },
      "histogram" : {
        "mean" : 10,
        "p75" : 10,
        "count" : 4,
        "median" : 10,
        "p95" : 10,
        "max" : 10,
        "p999" : 10,
        "variance" : 0,
        "stddev" : 0,
        "p99" : 10,
        "sum" : 40,
        "min" : 10
      }
    },
    "400" : {
      "meter" : {
        "mean" : 196.4806772592497,
        "15MinuteRate" : 0,
        "1MinuteRate" : 0,
        "count" : 1,
        "currentRate" : 196.4981671192174,
        "5MinuteRate" : 0
      },
      "histogram" : {
        "mean" : 10,
        "p75" : 10,
        "count" : 1,
        "median" : 10,
        "p95" : 10,
        "max" : 10,
        "p999" : 10,
        "variance" : null,
        "stddev" : 0,
        "p99" : 10,
        "sum" : 10,
        "min" : 10
      }
    },
    "lastResponseTime" : 10
  }
}

express_node_metrics.metrics.logInternalMetric(info, err)

Aggregate date using the info object. The aggregation is done by info.source three different sections:

global - all the data that related to the same source in one place.
statuses - the data that related to the same source split to success and fail execution.
methods - the data that related to the same source split by info.method.

each and every one of the sections is [Timer metric] (https://www.npmjs.com/package/measured#timers)

When executed with err, it will aggregate the info as failed execution in the statuses section.

Arguments

info (required) – the request object returned from Express middleware invocation
- source - The title for the metric data (for example southbound API destination).
- methodName - The method name that is measured.
- startTime - The start time of the measured method.
err – error if happened during the execution of the measured method.

express_node_metrics.metrics.addCustomGaugeMetric(metricName, metricValue)

This API allows to add custom metrics. All custom metrics will be aggregated according to the passed structure.

Arguments

metricName (required) – The metric name should be constructed with one of the following structures:
- <namespace>.<category>.<name>
- <namespace>.<category>.<sub category>.<name>
metricValue (required) – can be one of:
- A numeric metric value - this can be used for example when updating a metric value on certain events
- A function that returns a numeric value - this can be used when wanting to set a metric that should query a certain object to get the current value (for example process.uptime())

express_node_metrics.metrics.incrementCustomMetric(metricName)

This API allows to increment custom metrics - for example count all incoming requests. All custom metrics will be aggregated according to the passed structure.

Arguments

metricName (required) – The metric name should be constructed with one of the following structures:
- <namespace>.<category>.<name>
- <namespace>.<category>.<sub category>.<name>

express_node_metrics.metrics.decrementCustomMetric(metricName)

This API allows to decrement custom metrics - for example count all active incoming requests. This API can only be called after first using the incrementCustomMetric API (minimum value can be 0). All custom metrics will be aggregated according to the passed structure.

Arguments

metricName (required) – The metric name should be constructed with one of the following structures:
- <namespace>.<category>.<name>
- <namespace>.<category>.<sub category>.<name>

express_node_metrics.metrics.addCustomMeterMetric(metricName)

This API allows to add a custom Meter metric - things that are measured as events / interval.

The structure is:

mean: The average rate since the meter was started/ last reset.
count: Count of values added to the meter since was started/ last reset.
currentRate: The rate of the meter since the last reset.
1MinuteRate: The rate of the meter biased towards the last 1 minute.
5MinuteRate: The rate of the meter biased towards the last 5 minutes.
15MinuteRate: The rate of the meter biased towards the last 15 minutes.

Arguments

metricName (required) – The metric name should be constructed with one of the following structures:
- <namespace>.<category>.<name>
- <namespace>.<category>.<sub category>.<name>

How to Use With Docker

In order to use the package inside docker you should add node-gyp installation before 'npm insall' command:

RUN apk update && \
    # Install node-gyp dependencies
    apk add --no-cache make gcc g++ python && \
    # npm install
    npm install --production --silent && \
    # Uninstall node-gyp dependencies
    apk del make gcc g++ python

If you want to use your own fork while you waiting to accept merge request you need to also add git installation commands:

RUN apk update && \
    # Install git
    apk add --no-cache bash git openssh && \
    # Install node-gyp dependencies
    apk add --no-cache make gcc g++ python && \
    # npm install
    npm install --production --silent && \
    # Uninstall git
    apk del bash git openssh && \
    # Uninstall node-gyp dependencies
    apk del make gcc g++ python

Examples

Middleware

Express

var metricsMiddleware = require('express-node-metrics').middleware;
app.use(metricsMiddleware);

app.get('/users', function(req, res, next) {
    //Do Something
})
app.listen(3000);

Swagger-Express

// swaggerRouter configuration
var options = {
  controllers: './test/swaggerTest/controllers',
  useStubs: false
};

// The Swagger document (require it, build it programmatically, fetch it from a URL, ...)
var spec = fs.readFileSync('./test/swaggerTest/api/swagger.yaml', 'utf8');
var swaggerDoc = jsyaml.safeLoad(spec);

// Initialize the Swagger middleware
swaggerTools.initializeMiddleware(swaggerDoc, function (middleware) {
  
  // Interpret Swagger resources and attach metadata to request - must be first in swagger-tools middleware chain
  app.use(middleware.swaggerMetadata());

  app.use(metricsMiddleware);

  // Validate Swagger requests
  app.use(middleware.swaggerValidator());

  // Route validated requests to appropriate controller
  app.use(middleware.swaggerRouter(options));

  // Serve the Swagger documents and Swagger UI
  app.use(middleware.swaggerUi());

  // Start the server
  http.createServer(app).listen(serverPort, function () {
    console.log('Your server is listening on port %d (http://localhost:%d)', serverPort, serverPort);
    console.log('Swagger-ui is available on http://localhost:%d/docs', serverPort);
  });
});

Restify

var server = restify.createServer({
    name: 'Hello World'
});
server.use(metricsMiddleware);
server.use(restify.queryParser({ mapParams: true }));
server.use(restify.bodyParser({ mapParams: true }));
server.get('/hello/:user', function (req, res, next) {
    res.send('hello world');
});
server.get('/hello', function (req, res, next) {
    res.send('hello world');
});

server.listen(serverPort, function () {
    console.log('%s listening at %s', server.name, server.url);
});

Internal metrics

var middleware = require('express-node-metrics').metrics;
var stackTrace = require('stack-trace');
var kafka = require('../helpers/kafkaHelper');
var kafka_topic = process.env.KAFKA_TOPIC || "Sandbox_Apps_Storage_Audit";

function auditResponse(message, next) {
    var methodName = stackTrace.get()[0].getFunctionName();
    var startTime = Date.now();

    kafka.getResponseProducer().send([{
        topic: "kafka_topic",
        messages: message,
        partition: 0 // default 0 
    }], function (err, result) {
        metrics.logInternalMetric({ source: "kafka", startTime: startTime, methodName: methodName }, err);
        return next();
    });
}

Expose endpoint with all the metrics

'use strict'
var express = require("express");
var router = express.Router();
var metrics = require('express-node-metrics').metrics;

router.get('/', function (req, res) {
    res.send(metrics.getAll(req.query.reset));
});
router.get('/process', function (req, res) {
    res.send(metrics.processMetrics(req.query.reset));
});
router.get('/internal', function (req, res) {
    res.send(metrics.internalMetrics(req.query.reset));
});
router.get('/api', function (req, res) {
    res.send(metrics.apiMetrics(req.query.reset));
});

module.exports = router;

Running Tests

Using mocha, istanbul and mochawesome

npm test

Files

README.md

Latest commit

History

README.md

File metadata and controls

express-node-metrics

Table of Contents

Features

Install

API

express_node_metrics

express_node_metrics.middleware

express_node_metrics.metrics

express_node_metrics.metrics.getAll(reset)

Arguments

Returned function arguments

express_node_metrics.metrics.processMetrics(reset)

Arguments

Returned function arguments

express_node_metrics.metrics.apiMetrics(reset)

Arguments

Returned function arguments

express_node_metrics.metrics.internalMetrics(reset)

Arguments

Returned function arguments

express_node_metrics.metrics.addApiData(info)

Arguments

express_node_metrics.metrics.endPointMetrics(reset)

Arguments

Returned function arguments

express_node_metrics.metrics.logInternalMetric(info, err)

Arguments

express_node_metrics.metrics.addCustomGaugeMetric(metricName, metricValue)

Arguments

express_node_metrics.metrics.incrementCustomMetric(metricName)

Arguments

express_node_metrics.metrics.decrementCustomMetric(metricName)

Arguments

express_node_metrics.metrics.addCustomMeterMetric(metricName)

Arguments

How to Use With Docker

Examples

Middleware

Express

Swagger-Express

Restify

Internal metrics

Expose endpoint with all the metrics

Running Tests