From b7ee0ede2caa26ca02cb92521ab30cf41348edb8 Mon Sep 17 00:00:00 2001 From: Harry Chen Date: Thu, 25 Oct 2018 17:10:25 +0800 Subject: [PATCH] fix: use long replace number --- packages/metrics/package.json | 1 + packages/metrics/src/MetricsServerManager.ts | 2 + .../src/collect/CompactMetricsCollector.ts | 45 ++- .../metrics/src/collect/MetricsCollector.ts | 27 +- .../src/collect/NormalMetricsCollector.ts | 27 +- packages/metrics/src/common/domain.ts | 2 +- .../src/common/metrics/BucketCounter.ts | 37 +-- .../metrics/src/common/metrics/FastCompass.ts | 42 +-- .../src/common/metrics/LongBucketCounter.ts | 133 ++++++++ packages/metrics/src/common/metrics/Timer.ts | 2 +- packages/metrics/src/domain.ts | 300 ++++++++++++++++++ .../metrics/src/reporter/CustomReporter.ts | 3 +- .../reporter/FileMetricsManagerReporter.ts | 6 +- .../src/reporter/ScheduledMetricsReporter.ts | 6 +- .../test/unit/MetricsServerManager.test.ts | 2 +- .../test/unit/common/metrics/Counter.test.ts | 67 ++-- .../unit/common/metrics/FastCompass.test.ts | 36 ++- .../common/metrics/LongBucketCounter.test.ts | 53 ++++ .../FileMetricManagerReporter.test.ts | 23 +- 19 files changed, 684 insertions(+), 130 deletions(-) create mode 100644 packages/metrics/src/common/metrics/LongBucketCounter.ts create mode 100644 packages/metrics/test/unit/common/metrics/LongBucketCounter.test.ts diff --git a/packages/metrics/package.json b/packages/metrics/package.json index 474fc627..7e39d345 100644 --- a/packages/metrics/package.json +++ b/packages/metrics/package.json @@ -45,6 +45,7 @@ "koa": "^2.3.0", "koa-bodyparser": "^4.2.0", "koa-router": "^7.2.1", + "long": "^4.0.0", "mixin": "^0.2.0", "module-hook": "^2.1.0", "node-df": "^0.1.1", diff --git a/packages/metrics/src/MetricsServerManager.ts b/packages/metrics/src/MetricsServerManager.ts index 8344f03c..7e094725 100644 --- a/packages/metrics/src/MetricsServerManager.ts +++ b/packages/metrics/src/MetricsServerManager.ts @@ -337,6 +337,7 @@ export class MetricsServerManager extends AbstractIndicator implements MetricsMa result.set(MetricType.HISTOGRAM, metricRegistry.getHistograms(filter)); result.set(MetricType.METER, metricRegistry.getMeters(filter)); result.set(MetricType.TIMER, metricRegistry.getTimers(filter)); + result.set(MetricType.FASTCOMPASS, metricRegistry.getFastCompasses(filter)); return result; } @@ -350,6 +351,7 @@ export class MetricsServerManager extends AbstractIndicator implements MetricsMa result.set(MetricType.HISTOGRAM, allMetricsRegistry.getHistograms(filter)); result.set(MetricType.METER, allMetricsRegistry.getMeters(filter)); result.set(MetricType.TIMER, allMetricsRegistry.getTimers(filter)); + result.set(MetricType.FASTCOMPASS, allMetricsRegistry.getFastCompasses(filter)); return result; } diff --git a/packages/metrics/src/collect/CompactMetricsCollector.ts b/packages/metrics/src/collect/CompactMetricsCollector.ts index 0595c182..4db589e1 100644 --- a/packages/metrics/src/collect/CompactMetricsCollector.ts +++ b/packages/metrics/src/collect/CompactMetricsCollector.ts @@ -1,12 +1,8 @@ -import {MetricsCollector} from './MetricsCollector'; -import { BucketCounter, IFastCompass, MetricName } from '../common/index'; -import {MetricObject} from './MetricObject'; -import {ITimer} from '../common/metrics/Timer'; -import {IHistogram} from '../common/metrics/Histogram'; -import {ICounter} from '../common/metrics/Counter'; -import {IMeter} from '../common/metrics/Meter'; -import {Snapshot} from '../common/domain'; +import { MetricsCollector } from './MetricsCollector'; +import { BucketCounter, ICounter, IFastCompass, IHistogram, IMeter, ITimer, MetricName, Snapshot } from '../common'; +import { MetricObject } from './MetricObject'; +const BigNumber = require('long'); /** * 根据采集周期获取采集点，支持多个采集点的输出 */ @@ -17,7 +13,7 @@ export class CompactMetricsCollector extends MetricsCollector { let startTime = timestamp - this.reportInterval * 1000 + 1; let totalCounts = timer.getInstantCount(startTime); - for(let [time, metricValue] of totalCounts.entries()) { + for (let [ time, metricValue ] of totalCounts.entries()) { this.addMetricWithSuffix(name, 'bucket_count', metricValue, time, MetricObject.MetricType.DELTA, this.metricsCollectPeriodConfig.period(name.getMetricLevel())); } @@ -46,7 +42,7 @@ export class CompactMetricsCollector extends MetricsCollector { if (counter instanceof BucketCounter) { let totalCounts = ( counter).getBucketCounts(startTime); - for(let [time, metricValue] of totalCounts.entries()) { + for (let [ time, metricValue ] of totalCounts.entries()) { this.addMetricWithSuffix(name, 'bucket_count', metricValue, time, MetricObject.MetricType.DELTA, this.metricsCollectPeriodConfig.period(name.getMetricLevel())); } } @@ -60,7 +56,7 @@ export class CompactMetricsCollector extends MetricsCollector { let startTime = timestamp - this.reportInterval * 1000 + 1; let totalCounts = meter.getInstantCount(startTime); - for(let [time, metricValue] of totalCounts.entries()) { + for (let [ time, metricValue ] of totalCounts.entries()) { this.addMetricWithSuffix(name, 'bucket_count', metricValue, time, MetricObject.MetricType.DELTA, this.metricsCollectPeriodConfig.period(name.getMetricLevel())); } @@ -73,23 +69,24 @@ export class CompactMetricsCollector extends MetricsCollector { let bucketInterval = fastCompass.getBucketInterval(); let start = this.getNormalizedStartTime(timestamp, bucketInterval); - let totalCount = 0; - let totalRt = 0; - let successCount = 0; - let hitCount = -1; + let totalCount = new BigNumber(); + let totalRt = new BigNumber(); + let successCount = new BigNumber(); + let hitCount = new BigNumber(-1); let countPerCategory = fastCompass.getMethodCountPerCategory(start); - for (let [key, value] of countPerCategory.entries()) { + for (let [ key, value ] of countPerCategory.entries()) { if (value.has(start)) { - this.addMetricWithSuffix(name, key + '_bucket_count', value.get(start), start, + this.addMetricWithSuffix(name, key + '_bucket_count', value.get(start).toString(), start, MetricObject.MetricType.DELTA, bucketInterval); - totalCount += value.get(start); + + totalCount.add(value.get(start)); if ('success' === key) { - successCount += value.get(start); + successCount.add(value.get(start)); } if ('hit' === key) { hitCount = value.get(start); - successCount += value.get(start); + successCount.add(value.get(start)); } } else { this.addMetricWithSuffix(name, key + '_bucket_count', 0, start, @@ -99,12 +96,12 @@ export class CompactMetricsCollector extends MetricsCollector { for (let value of fastCompass.getMethodRtPerCategory(start).values()) { if (value.has(start)) { - totalRt += value.get(start); + totalRt.add(value.get(start)); } } - this.addMetricWithSuffix(name, 'bucket_count', totalCount, start, + this.addMetricWithSuffix(name, 'bucket_count', totalCount.toString(), start, MetricObject.MetricType.DELTA, bucketInterval); - this.addMetricWithSuffix(name, 'bucket_sum', totalRt, start, + this.addMetricWithSuffix(name, 'bucket_sum', totalRt.toString(), start, MetricObject.MetricType.DELTA, bucketInterval); this.addMetricWithSuffix(name, 'qps', this.rate(totalCount, bucketInterval), start, MetricObject.MetricType.GAUGE, bucketInterval); @@ -112,7 +109,7 @@ export class CompactMetricsCollector extends MetricsCollector { MetricObject.MetricType.GAUGE, bucketInterval); this.addMetricWithSuffix(name, 'success_rate', this.ratio(successCount, totalCount), start, MetricObject.MetricType.GAUGE, bucketInterval); - if (hitCount >= 0) { + if (hitCount.gte(0)) { this.addMetricWithSuffix(name, 'hit_rate', this.ratio(hitCount, successCount), start, MetricObject.MetricType.GAUGE, bucketInterval); } diff --git a/packages/metrics/src/collect/MetricsCollector.ts b/packages/metrics/src/collect/MetricsCollector.ts index e0d87776..9c3b8f5f 100644 --- a/packages/metrics/src/collect/MetricsCollector.ts +++ b/packages/metrics/src/collect/MetricsCollector.ts @@ -1,5 +1,6 @@ -import {MetricFilter, MetricName, MetricsCollectPeriodConfig} from '../common/index'; -import {CollectMetricType, MetricObject} from './MetricObject'; +import { MetricFilter, MetricName, MetricsCollectPeriodConfig } from '../common/index'; +import { CollectMetricType, MetricObject } from './MetricObject'; +import { Long } from '../domain'; export class MetricsCollector { protected metrics: Array = []; @@ -13,7 +14,7 @@ export class MetricsCollector { */ protected filter: MetricFilter; - constructor(options: { globalTags, rateFactor, durationFactor, filter?, reportInterval?}) { + constructor(options: { globalTags, rateFactor, durationFactor, filter?, reportInterval? }) { this.globalTags = options.globalTags; this.rateFactor = options.rateFactor; this.durationFactor = options.durationFactor; @@ -77,12 +78,24 @@ export class MetricsCollector { public rate(data, interval) { if (interval === 0) return 0.0; - return data / interval; + if (typeof interval !== 'number' && interval.isZero()) return 0.0; + + if (typeof data !== 'number') { + return data.div(interval).toString(); + } else { + return data / interval; + } } public ratio(data, total) { - if (data > total) return 1.0; - if (total === 0) return 0.0; - return 1.0 * data / total; + if (typeof data !== 'number') { + if (data.gt(total)) return 1.0; + if (total.eq(0)) return 0.0; + return data.div(total).toString(); + } else { + if (data > total) return 1.0; + if (total === 0) return 0.0; + return 1.0 * data / total; + } } } diff --git a/packages/metrics/src/collect/NormalMetricsCollector.ts b/packages/metrics/src/collect/NormalMetricsCollector.ts index 7b4c80e4..ce4cf5f4 100644 --- a/packages/metrics/src/collect/NormalMetricsCollector.ts +++ b/packages/metrics/src/collect/NormalMetricsCollector.ts @@ -2,6 +2,8 @@ import { MetricsCollector } from './MetricsCollector'; import { BucketCounter, ICounter, IFastCompass, IHistogram, IMeter, ITimer, MetricName, Snapshot } from '../common'; import { MetricObject } from './MetricObject'; +const BigNumber = require('long'); + export class NormalMetricsCollector extends MetricsCollector { collectTimer(name: MetricName, timer: ITimer, timestamp: number) { @@ -68,23 +70,24 @@ export class NormalMetricsCollector extends MetricsCollector { let bucketInterval = fastCompass.getBucketInterval(); let start = this.getNormalizedStartTime(timestamp, bucketInterval); - let totalCount = 0; - let totalRt = 0; - let successCount = 0; - let hitCount = -1; + let totalCount = new BigNumber(); + let totalRt = new BigNumber(); + let successCount = new BigNumber(); + let hitCount = new BigNumber(-1); let countPerCategory = fastCompass.getMethodCountPerCategory(start); for (let [ key, value ] of countPerCategory.entries()) { if (value.has(start)) { - this.addMetricWithSuffix(name, key + '_bucket_count', value.get(start), start, + this.addMetricWithSuffix(name, key + '_bucket_count', value.get(start).toString(), start, MetricObject.MetricType.DELTA, bucketInterval); - totalCount += value.get(start); + + totalCount.add(value.get(start)); if ('success' === key) { - successCount += value.get(start); + successCount.add(value.get(start)); } if ('hit' === key) { hitCount = value.get(start); - successCount += value.get(start); + successCount.add(value.get(start)); } } else { this.addMetricWithSuffix(name, key + '_bucket_count', 0, start, @@ -94,12 +97,12 @@ export class NormalMetricsCollector extends MetricsCollector { for (let value of fastCompass.getMethodRtPerCategory(start).values()) { if (value.has(start)) { - totalRt += value.get(start); + totalRt.add(value.get(start)); } } - this.addMetricWithSuffix(name, 'bucket_count', totalCount, start, + this.addMetricWithSuffix(name, 'bucket_count', totalCount.toString(), start, MetricObject.MetricType.DELTA, bucketInterval); - this.addMetricWithSuffix(name, 'bucket_sum', totalRt, start, + this.addMetricWithSuffix(name, 'bucket_sum', totalRt.toString(), start, MetricObject.MetricType.DELTA, bucketInterval); this.addMetricWithSuffix(name, 'qps', this.rate(totalCount, bucketInterval), start, MetricObject.MetricType.GAUGE, bucketInterval); @@ -107,7 +110,7 @@ export class NormalMetricsCollector extends MetricsCollector { MetricObject.MetricType.GAUGE, bucketInterval); this.addMetricWithSuffix(name, 'success_rate', this.ratio(successCount, totalCount), start, MetricObject.MetricType.GAUGE, bucketInterval); - if (hitCount >= 0) { + if (hitCount.gte(0)) { this.addMetricWithSuffix(name, 'hit_rate', this.ratio(hitCount, successCount), start, MetricObject.MetricType.GAUGE, bucketInterval); } diff --git a/packages/metrics/src/common/domain.ts b/packages/metrics/src/common/domain.ts index 93a4d975..9fd04d02 100644 --- a/packages/metrics/src/common/domain.ts +++ b/packages/metrics/src/common/domain.ts @@ -13,7 +13,7 @@ export interface Counting { * * @return the current count */ - getCount(): number; + getCount(): number | string; } diff --git a/packages/metrics/src/common/metrics/BucketCounter.ts b/packages/metrics/src/common/metrics/BucketCounter.ts index b85f9000..737f5e03 100644 --- a/packages/metrics/src/common/metrics/BucketCounter.ts +++ b/packages/metrics/src/common/metrics/BucketCounter.ts @@ -1,5 +1,4 @@ -import {BaseCounter, ICounter} from './Counter'; -import {MetricType} from '../MetricType'; +import { BaseCounter, ICounter } from './Counter'; /** * 提供分桶计数功能，每个桶统计一定时间间隔内的计数。 @@ -34,12 +33,12 @@ export interface IBucketCounter extends ICounter { getBucketInterval(); } -interface Bucket { +export interface Bucket { timestamp; count; } -class BucketDeque { +export class BucketDeque { private queue: Array = []; @@ -51,20 +50,24 @@ class BucketDeque { this.size = length; // init buckets for (let i = 0; i < length; i++) { - this.queue[i] = { - timestamp: -1, - count: 0, - }; + this.queue[ i ] = this.createQueueItem(); } } + protected createQueueItem(): Bucket { + return { + timestamp: -1, + count: 0, + }; + } + addLast(e: Bucket) { this.current = (this.current + 1) % this.size; - this.queue[this.current] = e; + this.queue[ this.current ] = e; } peek(): Bucket { - return this.queue[this.current]; + return this.queue[ this.current ]; } /** @@ -83,19 +86,19 @@ class BucketDeque { let length = this.queue.length - 1; let bucketList: Array = []; let startPos = this.current; - let startTs = this.queue[this.current].timestamp; + let startTs = this.queue[ this.current ].timestamp; if (startPos < 0) { startPos = 0; } for (let i = startPos; i >= 0 && startPos - i < length; i--) { - bucketList.push(this.queue[i]); + bucketList.push(this.queue[ i ]); } for (let i = length; i > startPos + 1; i--) { - if (this.queue[i].timestamp > startTs) { + if (this.queue[ i ].timestamp > startTs) { // the current index has been update during this iteration // therefore the data shall not be collected } else { - bucketList.push(this.queue[i]); + bucketList.push(this.queue[ i ]); } } return bucketList; @@ -105,12 +108,10 @@ class BucketDeque { export class BucketCounter extends BaseCounter implements IBucketCounter { - type = MetricType.COUNTER; - /** * 保存从创建开始累积的计数 */ - private totalCount: BaseCounter; + private totalCount: ICounter; /** * 保存最近N次的精确计数, 采用环形队列避免数据的挪动 @@ -178,7 +179,7 @@ export class BucketCounter extends BaseCounter implements IBucketCounter { } getCount() { - return this.totalCount.getCount(); + return this.totalCount.getCount(); } inc(n?) { diff --git a/packages/metrics/src/common/metrics/FastCompass.ts b/packages/metrics/src/common/metrics/FastCompass.ts index 4ad2a157..a6719311 100644 --- a/packages/metrics/src/common/metrics/FastCompass.ts +++ b/packages/metrics/src/common/metrics/FastCompass.ts @@ -1,6 +1,9 @@ import { MetricType } from '../MetricType'; import { Metric } from '../domain'; -import { BucketCounter } from './BucketCounter'; +import { LongBucketCounter } from './LongBucketCounter'; +import { Long } from '../../domain'; + +const BigNumber = require("long"); export interface IFastCompass extends Metric { @@ -20,7 +23,7 @@ export interface IFastCompass extends Metric { * return method count per bucket per category * @return */ - getMethodCountPerCategory(startTime?): Map>; + getMethodCountPerCategory(startTime?): Map>; /** @@ -28,14 +31,14 @@ export interface IFastCompass extends Metric { * return method execution time and count per bucket per category * @return */ - getMethodRtPerCategory(startTime?): Map>; + getMethodRtPerCategory(startTime?): Map>; /** * 对于每个子类别，返回每个统计间隔的执行总时间和次数，按位分离操作放到下一层进行 * return method execution time and count per bucket per category * @return */ - getCountAndRtPerCategory(startTime): Map>; + getCountAndRtPerCategory(startTime): Map>; /** * 获取统计间隔 @@ -59,14 +62,14 @@ const COUNT_OFFSET = 38; * The base number of count that is added to total rt, * to derive a number which will be added to {@link LongAdder} */ -const COUNT_BASE = 1 << 38; +const COUNT_BASE = new BigNumber(1).shiftLeft(38); /** * 总数和此数进行二进制与得到总rt统计 * The base number is used to do BITWISE AND operation with the value of {@link LongAdder} * to derive the total number of execution time */ -const RT_BITWISE_AND_BASE = (1 << 38) - 1; +const RT_BITWISE_AND_BASE = new BigNumber(1).shiftLeft(38).sub(1); const MAX_SUBCATEGORY_SIZE = 20; @@ -86,13 +89,12 @@ export class BaseFastCompass implements IFastCompass { bucketInterval; numberOfBuckets; maxCategoryCount; - subCategories: Map; + subCategories: Map = new Map(); - constructor(bucketInterval, numberOfBuckets = DEFAULT_BUCKET_COUNT, maxCategoryCount = MAX_SUBCATEGORY_SIZE) { + constructor(bucketInterval = 60, numberOfBuckets = DEFAULT_BUCKET_COUNT, maxCategoryCount = MAX_SUBCATEGORY_SIZE) { this.bucketInterval = bucketInterval; this.numberOfBuckets = numberOfBuckets; this.maxCategoryCount = maxCategoryCount; - this.subCategories = new Map(); } @@ -105,18 +107,18 @@ export class BaseFastCompass implements IFastCompass { // ignore if maxCategoryCount is exceeded, no exception will be thrown return; } - this.subCategories.set(subCategory, new BucketCounter(this.bucketInterval, this.numberOfBuckets, false)); + this.subCategories.set(subCategory, new LongBucketCounter(this.bucketInterval, this.numberOfBuckets, false)); } - let data = COUNT_BASE + duration; + let data = COUNT_BASE.add(duration); this.subCategories.get(subCategory).update(data); } - getMethodCountPerCategory(startTime = 0): Map> { - let countPerCategory: Map> = new Map(); + getMethodCountPerCategory(startTime = 0) { + let countPerCategory: Map> = new Map(); for (let [ key, value ] of this.subCategories.entries()) { - let bucketCount: Map = new Map(); + let bucketCount: Map = new Map(); for (let [ innerKey, innerValue ] of value.getBucketCounts(startTime).entries()) { - bucketCount.set(innerKey, innerValue >> COUNT_OFFSET); + bucketCount.set(innerKey, innerValue.shiftRight(COUNT_OFFSET)); } countPerCategory.set(key, bucketCount); } @@ -124,11 +126,11 @@ export class BaseFastCompass implements IFastCompass { } getMethodRtPerCategory(startTime = 0) { - let rtPerCategory: Map> = new Map(); + let rtPerCategory: Map> = new Map(); for (let [ key, value ] of this.subCategories.entries()) { - let bucketCount: Map = new Map(); + let bucketCount: Map = new Map(); for (let [ innerKey, innerValue ] of value.getBucketCounts(startTime).entries()) { - bucketCount.set(innerKey, innerValue & RT_BITWISE_AND_BASE); + bucketCount.set(innerKey, innerValue.and(RT_BITWISE_AND_BASE)); } rtPerCategory.set(key, bucketCount); } @@ -141,9 +143,9 @@ export class BaseFastCompass implements IFastCompass { } getCountAndRtPerCategory(startTime = 0) { - let countAndRtPerCategory: Map> = new Map(); + let countAndRtPerCategory: Map> = new Map(); for (let [ key, value ] of this.subCategories.entries()) { - let bucketCount: Map = new Map(); + let bucketCount: Map = new Map(); for (let [ innerKey, innerValue ] of value.getBucketCounts(startTime).entries()) { bucketCount.set(innerKey, innerValue); } diff --git a/packages/metrics/src/common/metrics/LongBucketCounter.ts b/packages/metrics/src/common/metrics/LongBucketCounter.ts new file mode 100644 index 00000000..eb0496c5 --- /dev/null +++ b/packages/metrics/src/common/metrics/LongBucketCounter.ts @@ -0,0 +1,133 @@ +import { Bucket, BucketDeque, IBucketCounter } from './BucketCounter'; +import { ICounter } from './Counter'; +import { MetricType } from '../MetricType'; +import { Long } from '../../domain'; + +const BigNumber = require("long"); + +export class LongBucketDeque extends BucketDeque { + protected createQueueItem(): Bucket { + return { + timestamp: -1, + count: new BigNumber(), + }; + } +} + +export class LongBaseCounter implements ICounter { + + count: Long = new BigNumber(); + type = MetricType.COUNTER; + + dec(n: number = 1) { + this.count = this.count.sub(n); + } + + inc(n: number = 1) { + this.count = this.count.add(n); + } + + getCount(): string { + return this.count.toString(); + } + + clear() { + this.count = new BigNumber(); + } +} + +export class LongBucketCounter extends LongBaseCounter implements IBucketCounter { + + /** + * 保存从创建开始累积的计数 + */ + private totalCount: ICounter = new LongBaseCounter(); + + /** + * 保存最近N次的精确计数, 采用环形队列避免数据的挪动 + */ + private buckets: LongBucketDeque; + + /** + * 是否更新总次数 + */ + private updateTotalCount: boolean = false; + + /** + * 每一次精确计数的之间的时间间隔，单位秒 + * 只能是 1，5，10, 30, 60 这几个数字 + */ + private interval; + + constructor(interval = 1, numberOfBucket = 10, updateTotalCount = true) { + super(); + this.interval = interval; + this.buckets = new LongBucketDeque(numberOfBucket + 1); + this.updateTotalCount = updateTotalCount; + } + + getBucketCounts(startTime = 0): Map { + let counts: Map = new Map(); + let curTs = this.calculateCurrentTimestamp(Date.now()); + + for (let bucket of this.buckets.getBucketList()) { + if (1000 * bucket.timestamp >= startTime && bucket.timestamp <= curTs) { + counts.set(bucket.timestamp * 1000, bucket.count); + } + } + return counts; + } + + getBucketCountsValues(startTime = 0): Map { + const maps = this.getBucketCounts(startTime); + const newMaps = new Map(); + + maps.forEach((value, key) => { + newMaps.set(key, value.toString()); + }); + return newMaps; + } + + private calculateCurrentTimestamp(timestamp: number) { + // transform to seconds and discard fractional part + return Math.floor(Math.floor(timestamp / 1000) / this.interval) * this.interval; + } + + getBucketInterval() { + return this.interval; + } + + update(n = 1) { + if (this.updateTotalCount) { + this.totalCount.inc(n); + } + + let curTs = this.calculateCurrentTimestamp(Date.now()); + let lastBucket = this.buckets.peek(); + + if (curTs > lastBucket.timestamp) { + // create a new bucket and evict the oldest one + let newBucket: Bucket = { + count: new BigNumber(), + timestamp: curTs + }; + + this.buckets.addLast(newBucket); + lastBucket = newBucket; + } + lastBucket.count = (lastBucket.count).add(n); + } + + getCount(): string { + return this.totalCount.getCount(); + } + + inc(n?) { + this.update(n); + } + + dec(n = 1) { + this.update(-n); + } + +} diff --git a/packages/metrics/src/common/metrics/Timer.ts b/packages/metrics/src/common/metrics/Timer.ts index a561aa97..b005f6ea 100644 --- a/packages/metrics/src/common/metrics/Timer.ts +++ b/packages/metrics/src/common/metrics/Timer.ts @@ -35,7 +35,7 @@ export class BaseTimer implements ITimer { } getCount() { - return this.histogram.getCount(); + return this.histogram.getCount(); } getFifteenMinuteRate() { diff --git a/packages/metrics/src/domain.ts b/packages/metrics/src/domain.ts index 5c89b1ba..59868dff 100644 --- a/packages/metrics/src/domain.ts +++ b/packages/metrics/src/domain.ts @@ -192,3 +192,303 @@ export interface TracerReport { report(); getValue(); } + +export interface Long { + /** + * Returns the sum of this and the specified Long. + */ + add(addend: number | Long | string): Long; + + /** + * Returns the bitwise AND of this Long and the specified. + */ + and(other: Long | number | string): Long; + + /** + * Compares this Long's value with the specified's. + */ + compare(other: Long | number | string): number; + + /** + * Compares this Long's value with the specified's. + */ + comp(other: Long | number | string): number; + + /** + * Returns this Long divided by the specified. + */ + divide(divisor: Long | number | string): Long; + + /** + * Returns this Long divided by the specified. + */ + div(divisor: Long | number | string): Long; + + /** + * Tests if this Long's value equals the specified's. + */ + equals(other: Long | number | string): boolean; + + /** + * Tests if this Long's value equals the specified's. + */ + eq(other: Long | number | string): boolean; + + /** + * Gets the high 32 bits as a signed integer. + */ + getHighBits(): number; + + /** + * Gets the high 32 bits as an unsigned integer. + */ + getHighBitsUnsigned(): number; + + /** + * Gets the low 32 bits as a signed integer. + */ + getLowBits(): number; + + /** + * Gets the low 32 bits as an unsigned integer. + */ + getLowBitsUnsigned(): number; + + /** + * Gets the number of bits needed to represent the absolute value of this Long. + */ + getNumBitsAbs(): number; + + /** + * Tests if this Long's value is greater than the specified's. + */ + greaterThan(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is greater than the specified's. + */ + gt(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is greater than or equal the specified's. + */ + greaterThanOrEqual(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is greater than or equal the specified's. + */ + gte(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is greater than or equal the specified's. + */ + ge(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is even. + */ + isEven(): boolean; + + /** + * Tests if this Long's value is negative. + */ + isNegative(): boolean; + + /** + * Tests if this Long's value is odd. + */ + isOdd(): boolean; + + /** + * Tests if this Long's value is positive. + */ + isPositive(): boolean; + + /** + * Tests if this Long's value equals zero. + */ + isZero(): boolean; + + /** + * Tests if this Long's value equals zero. + */ + eqz(): boolean; + + /** + * Tests if this Long's value is less than the specified's. + */ + lessThan(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is less than the specified's. + */ + lt(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is less than or equal the specified's. + */ + lessThanOrEqual(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is less than or equal the specified's. + */ + lte(other: Long | number | string): boolean; + + /** + * Tests if this Long's value is less than or equal the specified's. + */ + le(other: Long | number | string): boolean; + + /** + * Returns this Long modulo the specified. + */ + modulo(other: Long | number | string): Long; + + /** + * Returns this Long modulo the specified. + */ + mod(other: Long | number | string): Long; + + /** + * Returns this Long modulo the specified. + */ + rem(other: Long | number | string): Long; + + /** + * Returns the product of this and the specified Long. + */ + multiply(multiplier: Long | number | string): Long; + + /** + * Returns the product of this and the specified Long. + */ + mul(multiplier: Long | number | string): Long; + + /** + * Negates this Long's value. + */ + negate(): Long; + + /** + * Negates this Long's value. + */ + neg(): Long; + + /** + * Returns the bitwise NOT of this Long. + */ + not(): Long; + + /** + * Tests if this Long's value differs from the specified's. + */ + notEquals(other: Long | number | string): boolean; + + /** + * Tests if this Long's value differs from the specified's. + */ + neq(other: Long | number | string): boolean; + + /** + * Tests if this Long's value differs from the specified's. + */ + ne(other: Long | number | string): boolean; + + /** + * Returns the bitwise OR of this Long and the specified. + */ + or(other: Long | number | string): Long; + + /** + * Returns this Long with bits shifted to the left by the given amount. + */ + shiftLeft(numBits: number | Long): Long; + + /** + * Returns this Long with bits shifted to the left by the given amount. + */ + shl(numBits: number | Long): Long; + + /** + * Returns this Long with bits arithmetically shifted to the right by the given amount. + */ + shiftRight(numBits: number | Long): Long; + + /** + * Returns this Long with bits arithmetically shifted to the right by the given amount. + */ + shr(numBits: number | Long): Long; + + /** + * Returns this Long with bits logically shifted to the right by the given amount. + */ + shiftRightUnsigned(numBits: number | Long): Long; + + /** + * Returns this Long with bits logically shifted to the right by the given amount. + */ + shru(numBits: number | Long): Long; + + /** + * Returns this Long with bits logically shifted to the right by the given amount. + */ + shr_u(numBits: number | Long): Long; + + /** + * Returns the difference of this and the specified Long. + */ + subtract(subtrahend: number | Long | string): Long; + + /** + * Returns the difference of this and the specified Long. + */ + sub(subtrahend: number | Long |string): Long; + + /** + * Converts the Long to a 32 bit integer, assuming it is a 32 bit integer. + */ + toInt(): number; + + /** + * Converts the Long to a the nearest floating-point representation of this value (double, 53 bit mantissa). + */ + toNumber(): number; + + /** + * Converts this Long to its byte representation. + */ + + toBytes(le?: boolean): number[]; + + /** + * Converts this Long to its little endian byte representation. + */ + + toBytesLE(): number[]; + + /** + * Converts this Long to its big endian byte representation. + */ + + toBytesBE(): number[]; + + /** + * Converts this Long to signed. + */ + toSigned(): Long; + + /** + * Converts the Long to a string written in the specified radix. + */ + toString(radix?: number): string; + + /** + * Converts this Long to unsigned. + */ + toUnsigned(): Long; + + /** + * Returns the bitwise XOR of this Long and the given one. + */ + xor(other: Long | number | string): Long; +} diff --git a/packages/metrics/src/reporter/CustomReporter.ts b/packages/metrics/src/reporter/CustomReporter.ts index 38d89cae..e43be070 100644 --- a/packages/metrics/src/reporter/CustomReporter.ts +++ b/packages/metrics/src/reporter/CustomReporter.ts @@ -42,7 +42,8 @@ export abstract class CustomReporter implements Reporter { counters: categoryMetrics.get(MetricType.COUNTER), histograms: categoryMetrics.get(MetricType.HISTOGRAM), meters: categoryMetrics.get(MetricType.METER), - timers: categoryMetrics.get(MetricType.TIMER) + timers: categoryMetrics.get(MetricType.TIMER), + fastCompassees: categoryMetrics.get(MetricType.FASTCOMPASS) }; } diff --git a/packages/metrics/src/reporter/FileMetricsManagerReporter.ts b/packages/metrics/src/reporter/FileMetricsManagerReporter.ts index 018a2d43..c2a24594 100644 --- a/packages/metrics/src/reporter/FileMetricsManagerReporter.ts +++ b/packages/metrics/src/reporter/FileMetricsManagerReporter.ts @@ -39,7 +39,7 @@ export class FileMetricsManagerReporter extends ScheduledMetricsReporter { } async report(metricsData) { - let {gauges, counters, histograms, meters, timers} = metricsData; + let {gauges, counters, histograms, meters, timers, fastCompasses} = metricsData; const timestamp = Date.now(); const collector = new this.collectorCls({ @@ -74,6 +74,10 @@ export class FileMetricsManagerReporter extends ScheduledMetricsReporter { collector.collectTimer(MetricName.parseKey(key), timer, timestamp); } + for (let [key, fastCompass] of fastCompasses.entries()) { + collector.collectFastCompass(MetricName.parseKey(key), fastCompass, timestamp); + } + try { // 只显示 metrics 的 report for (let metricObject of collector.build()) { diff --git a/packages/metrics/src/reporter/ScheduledMetricsReporter.ts b/packages/metrics/src/reporter/ScheduledMetricsReporter.ts index ec1c7a39..44ab90a2 100644 --- a/packages/metrics/src/reporter/ScheduledMetricsReporter.ts +++ b/packages/metrics/src/reporter/ScheduledMetricsReporter.ts @@ -5,7 +5,7 @@ import { BaseHistogram, BaseMeter, BaseTimer, - MetricType, + MetricType, BaseFastCompass, } from '../common/index'; const debug = require('debug')('pandora:metrics:schedule-reporter'); @@ -44,7 +44,8 @@ export abstract class ScheduledMetricsReporter implements Reporter { counters: categoryMetrics.get(MetricType.COUNTER), histograms: categoryMetrics.get(MetricType.HISTOGRAM), meters: categoryMetrics.get(MetricType.METER), - timers: categoryMetrics.get(MetricType.TIMER) + timers: categoryMetrics.get(MetricType.TIMER), + fastCompasses: categoryMetrics.get(MetricType.FASTCOMPASS) } ); } catch (err) { @@ -60,6 +61,7 @@ export abstract class ScheduledMetricsReporter implements Reporter { histograms: Map, meters: Map, timers: Map, + fastCompasses: Map }); stop() { diff --git a/packages/metrics/test/unit/MetricsServerManager.test.ts b/packages/metrics/test/unit/MetricsServerManager.test.ts index 4b594785..e43290ac 100644 --- a/packages/metrics/test/unit/MetricsServerManager.test.ts +++ b/packages/metrics/test/unit/MetricsServerManager.test.ts @@ -133,7 +133,7 @@ describe('/test/unit/MetricsServerManager.test.ts', () => { expect(server.listMetricNamesByGroup().size > 0).to.be.true; expect(server.listMetricNamesByGroup().get('middleware').length).to.equal(5); - expect(server.getAllCategoryMetrics().size).to.equal(5); + expect(server.getAllCategoryMetrics().size).to.equal(6); }); diff --git a/packages/metrics/test/unit/common/metrics/Counter.test.ts b/packages/metrics/test/unit/common/metrics/Counter.test.ts index 142c9720..1e47595e 100644 --- a/packages/metrics/test/unit/common/metrics/Counter.test.ts +++ b/packages/metrics/test/unit/common/metrics/Counter.test.ts @@ -1,31 +1,62 @@ import {expect} from 'chai'; import {BaseCounter} from '../../../../src/common/metrics/Counter'; +import { LongBaseCounter } from '../../../../src/common/metrics/LongBucketCounter'; describe('/test/unit/common/metrics/Counter.test.ts', () => { - const counter = new BaseCounter(); + describe('BaseCounter', () => { + const counter = new BaseCounter(); - it('startsAtZero', () => { - expect(counter.getCount()).to.equal(0); - }); + it('startsAtZero', () => { + expect(counter.getCount()).to.equal(0); + }); - it('incrementsByOne', () => { - counter.inc(); - expect(counter.getCount()).to.equal(1); - }); + it('incrementsByOne', () => { + counter.inc(); + expect(counter.getCount()).to.equal(1); + }); - it('incrementsByAnArbitraryDelta', () => { - counter.inc(12); - expect(counter.getCount()).to.equal(13); - }); + it('incrementsByAnArbitraryDelta', () => { + counter.inc(12); + expect(counter.getCount()).to.equal(13); + }); - it('decrementsByOne', () => { - counter.dec(); - expect(counter.getCount()).to.equal(12); + it('decrementsByOne', () => { + counter.dec(); + expect(counter.getCount()).to.equal(12); + }); + + it('decrementsByAnArbitraryDelta', () => { + counter.dec(12); + expect(counter.getCount()).to.equal(0); + }); }); - it('decrementsByAnArbitraryDelta', () => { - counter.dec(12); - expect(counter.getCount()).to.equal(0); + describe('LongBaseCounter', () => { + const counter = new LongBaseCounter(); + + it('startsAtZero', () => { + expect(counter.getCount()).to.equal('0'); + }); + + it('incrementsByOne', () => { + counter.inc(); + expect(counter.getCount()).to.equal('1'); + }); + + it('incrementsByAnArbitraryDelta', () => { + counter.inc(12); + expect(counter.getCount()).to.equal('13'); + }); + + it('decrementsByOne', () => { + counter.dec(); + expect(counter.getCount()).to.equal('12'); + }); + + it('decrementsByAnArbitraryDelta', () => { + counter.dec(12); + expect(counter.getCount()).to.equal('0'); + }); }); }); diff --git a/packages/metrics/test/unit/common/metrics/FastCompass.test.ts b/packages/metrics/test/unit/common/metrics/FastCompass.test.ts index 5414c243..56c84f39 100644 --- a/packages/metrics/test/unit/common/metrics/FastCompass.test.ts +++ b/packages/metrics/test/unit/common/metrics/FastCompass.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { BaseFastCompass } from '../../../../src/common/metrics/FastCompass'; - -describe('/test/unit/metrics/common/metrics/FastCompass.test.ts', () => { +const BigNumber = require('long'); +describe('/test/unit/common/metrics/FastCompass.test.ts', () => { it('testFastCompass', async () => { @@ -12,37 +12,39 @@ describe('/test/unit/metrics/common/metrics/FastCompass.test.ts', () => { // verify count expect(fastCompass.getMethodCountPerCategory().has('success')).to.ok; - expect(Array.from(fastCompass.getMethodCountPerCategory(0).get('success').values())[ 0 ]).to.equal(2); + expect(Array.from(fastCompass.getMethodCountPerCategory(0).get('success').values())[ 0 ].toString()).to.equal('2'); expect(fastCompass.getMethodCountPerCategory().has('error')).to.ok; - expect(Array.from(fastCompass.getMethodCountPerCategory(0).get('error').values())[ 0 ]).to.equal(1); + expect(Array.from(fastCompass.getMethodCountPerCategory(0).get('error').values())[ 0 ].toString()).to.equal('1'); // verify rt expect(fastCompass.getMethodRtPerCategory().has('success')).to.ok; - expect(Array.from(fastCompass.getMethodRtPerCategory(0).get('success').values())[ 0 ]).to.equal(5); + expect(Array.from(fastCompass.getMethodRtPerCategory(0).get('success').values())[ 0 ].toString()).to.equal('5'); expect(fastCompass.getMethodRtPerCategory().has('error')).to.ok; - expect(Array.from(fastCompass.getMethodRtPerCategory(0).get('error').values())[ 0 ]).to.equal(4); + expect(Array.from(fastCompass.getMethodRtPerCategory(0).get('error').values())[ 0 ].toString()).to.equal('4'); // total count - let totalCount = Array.from(fastCompass.getMethodCountPerCategory(0).get('success').values())[ 0 ] + - Array.from(fastCompass.getMethodCountPerCategory(0).get('error').values())[ 0 ]; - expect(totalCount).to.equal(3); + let totalCount = (Array.from(fastCompass.getMethodCountPerCategory(0).get('success').values())[ 0 ].add( + Array.from(fastCompass.getMethodCountPerCategory(0).get('error').values())[ 0 ]).toString()); + expect(totalCount).to.equal('3'); // average rt - let avgRt = (Array.from(fastCompass.getMethodRtPerCategory(0).get('success').values())[ 0 ] + - Array.from(fastCompass.getMethodRtPerCategory(0).get('error').values())[ 0 ]) / totalCount; - expect(avgRt).to.equal(3); + let avgRt = (Array.from(fastCompass.getMethodRtPerCategory(0).get('success').values())[ 0 ].add( + Array.from(fastCompass.getMethodRtPerCategory(0).get('error').values())[ 0 ])).div(totalCount); + expect(avgRt.toString()).to.equal('3'); // verify count and rt expect(fastCompass.getCountAndRtPerCategory().has('success')).to.ok; - expect(Array.from(fastCompass.getCountAndRtPerCategory(0).get('success').values())[ 0 ]).to.equal((2 << 38) + 5); + expect(Array.from(fastCompass.getCountAndRtPerCategory(0).get('success').values())[ 0 ].toString()).to.equal((new BigNumber(2).shiftLeft(38)).add(5).toString()); expect(fastCompass.getCountAndRtPerCategory().has('error')).to.ok; - expect(Array.from(fastCompass.getCountAndRtPerCategory(0).get('error').values())[ 0 ]).to.equal((1 << 38) + 4); + expect(Array.from(fastCompass.getCountAndRtPerCategory(0).get('error').values())[ 0 ].toString()).to.equal((new BigNumber(1).shiftLeft(38)).add(4).toString()); }); it('testBinaryAdd', () => { - let a1 = (1 << 38) + 10; - let a2 = (1 << 38) + 20; - expect((a1 + a2) >> 38).to.equal(2); + let a1 = new BigNumber(1).shiftLeft(38).add(10); + let a2 = new BigNumber(1).shiftLeft(38).add(20); + expect(a1.add(a2).shiftRight(38).toString()).to.equal('2'); + expect(new BigNumber(1).shiftLeft(38).toString()).to.equal('274877906944'); + expect(new BigNumber(-1, -1).toString()).to.equal(new BigNumber(1).sub(2).toString()); }); it('testMaxSubCategoryCount', () => { diff --git a/packages/metrics/test/unit/common/metrics/LongBucketCounter.test.ts b/packages/metrics/test/unit/common/metrics/LongBucketCounter.test.ts new file mode 100644 index 00000000..896ce3fc --- /dev/null +++ b/packages/metrics/test/unit/common/metrics/LongBucketCounter.test.ts @@ -0,0 +1,53 @@ +import { expect } from 'chai'; +import { LongBucketCounter } from '../../../../src/common/metrics/LongBucketCounter'; + +describe('/test/unit/common/metrics/LongBucketCounter.test.ts', () => { + + it('testSingleUpdate', async () => { + const bucketCounter = new LongBucketCounter(1, 5); + for (let k = 1; k <= 7; k++) { + for (let i = 0; i < k * 10; i++) { + bucketCounter.update(); + } + + await new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 1000); + }); + } + let expected = [ '70', '60', '50', '40', '30' ]; + expect(Array.from(bucketCounter.getBucketCountsValues().values())).to.deep.equal(expected); + }); + + it('testLatestIndexAtFirst', async () => { + const bucketCounter = new LongBucketCounter(1, 5); + for (let k = 1; k <= 6; k++) { + for (let i = 0; i < k * 10; i++) { + bucketCounter.update(); + } + + await new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, 1000); + }); + } + let expected = [ '60', '50', '40', '30', '20' ]; + expect(Array.from(bucketCounter.getBucketCountsValues().values())).to.deep.equal(expected); + }); + + it('testUpdateTotalCount', () => { + const bucketCounter = new LongBucketCounter(10, 10, true); + bucketCounter.update(); + bucketCounter.update(); + expect(bucketCounter.getCount()).to.equal('2'); + }); + + it('testNotUpdateTotalCount', () => { + const bucketCounter = new LongBucketCounter(10, 10, false); + bucketCounter.update(); + bucketCounter.update(); + expect(bucketCounter.getCount()).to.equal('0'); + }); +}); diff --git a/packages/metrics/test/unit/reporter/FileMetricManagerReporter.test.ts b/packages/metrics/test/unit/reporter/FileMetricManagerReporter.test.ts index 48efc8c8..00dee491 100644 --- a/packages/metrics/test/unit/reporter/FileMetricManagerReporter.test.ts +++ b/packages/metrics/test/unit/reporter/FileMetricManagerReporter.test.ts @@ -1,8 +1,16 @@ -import {expect} from 'chai'; -import {BaseCounter, MetricName, BaseHistogram, BaseTimer, BaseMeter, BaseGauge} from '../../../src/common/index'; -import {MetricsServerManager} from '../../../src/MetricsServerManager'; -import {FileMetricsManagerReporter} from '../../../src/reporter/FileMetricsManagerReporter'; -import {join} from 'path'; +import { expect } from 'chai'; +import { + BaseCounter, + BaseFastCompass, + BaseGauge, + BaseHistogram, + BaseMeter, + BaseTimer, + MetricName +} from '../../../src/common'; +import { FileMetricsManagerReporter, MetricsServerManager } from '../../../src'; +import { join } from 'path'; + const fs = require('fs'); const os = require('os'); @@ -11,7 +19,7 @@ describe('/test/unit/reporter/FileMetricManagerReporter.test.ts', () => { const metricsPath = join(os.tmpdir(), 'pandorajs/metrics.log'); before(() => { - if(fs.existsSync(metricsPath)) { + if (fs.existsSync(metricsPath)) { fs.unlinkSync(metricsPath); } }); @@ -30,7 +38,8 @@ describe('/test/unit/reporter/FileMetricManagerReporter.test.ts', () => { manager.register('test', MetricName.build('reporter.register.histogram'), new BaseHistogram()); manager.register('test', MetricName.build('reporter.register.timer'), new BaseTimer()); manager.register('test', MetricName.build('reporter.register.meter'), new BaseMeter()); - const reporter = new FileMetricsManagerReporter(null , {}); + manager.register('test', MetricName.build('reporter.register.fastCompass'), new BaseFastCompass()); + const reporter = new FileMetricsManagerReporter(null, {}); reporter.setMetricsManager(manager); reporter.start(0.4);