Merge pull request #963 from tasha-urbancic/feature/preserve-test-name

Add preserveTestName CLI flag to remove partition and browser
ember-cli · Jan 19, 2023 · c48bc2f · c48bc2f
2 parents d1b2521 + 79de2c6
commit c48bc2f
Show file tree

Hide file tree

Showing 9 changed files with 256 additions and 70 deletions.
diff --git a/README.md b/README.md
@@ -24,6 +24,7 @@ The [documentation website](https://ember-cli.github.io/ember-exam/) contains ex
     + [Split Test Parallelization](#split-test-parallelization)
   * [Test Load Balancing](#test-load-balancing)
       - [Test Failure Reproduction](#test-failure-reproduction)
+  * [Preserve Test Name](#preserve-test-name)
 - [Advanced Configuration](#advanced-configuration)
   * [Ember Try & CI Integration](#ember-try--ci-integration)
   * [Test Suite Segmentation](#test-suite-segmentation)
@@ -375,6 +376,33 @@ $ ember exam --replay-execution=test-execution-000000.json
 3. You must be using `qunit` version 2.8.0 or greater for this feature to work properly.
 4. This feature is not currently supported by Mocha.
 
+#### Preserve Test Name
+
+When using `--split` and/or `--load-balance` the output will look something like:
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=3 --load-balance
+ok 1 Chrome 66.0 - Exam Partition 1 - browser Id 1 - some test
+ok 2 Chrome 66.0 - Exam Partition 1 - browser Id 2 - another test
+ok 3 Chrome 66.0 - Exam Partition 1 - browser Id 3 - some the other test
+```
+However, if you change the amount of parallelization, or randomize across partitions, the output will change for the same test, which may be an issue if you are tracking test insights over time. 
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=2 --load-balance
+ok 1 Chrome 66.0 - Exam Partition 1 - browser Id 2 - some test
+ok 2 Chrome 66.0 - Exam Partition 1 - browser Id 1 - another test
+ok 3 Chrome 66.0 - Exam Partition 1 - browser Id 2 - some the other test
+```
+You can add `--preserve-test-name` to remove the dynamic segments of the output (partition and browser) to ensure the output test names are always the same.
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=3 --load-balance --preserve-test-name
+ok 1 Chrome 66.0 - some test
+ok 2 Chrome 66.0 - another test
+ok 3 Chrome 66.0 - some the other test
+```
+
 ## Advanced Configuration
 
 Ember Exam does its best to allow you to run your test suite in a way that is effective for your individual needs. To that end, there are lots of advanced ways to configure your setup by integrating with other aspects of the Ember testing environment. The following sections will cover a few of the more common scenarios.

diff --git a/addon-test-support/-private/patch-testem-output.js b/addon-test-support/-private/patch-testem-output.js
@@ -15,7 +15,11 @@ export function updateTestName(urlParams, testName) {
   const partition = urlParams.get('partition') || 1;
   const browser = urlParams.get('browser') || 1;
 
-  if (split && loadBalance) {
+  const preserveTestName = !!urlParams.get('preserveTestName');
+
+  if (preserveTestName) {
+    return testName;
+  } else if (split && loadBalance) {
     testName = `Exam Partition ${partition} - Browser Id ${browser} - ${testName}`;
   } else if (split) {
     testName = `Exam Partition ${partition} - ${testName}`;

diff --git a/lib/commands/exam.js b/lib/commands/exam.js
@@ -43,6 +43,14 @@ module.exports = TestCommand.extend({
       description:
         'Load balance test modules. Test modules will be sorted by weight from slowest (acceptance) to fastest (eslint).',
     },
+    {
+      name: 'preserve-test-name',
+      type: Boolean,
+      default: false,
+      aliases: ['ptn'],
+      description:
+        'Preserve the test name when using load balance or split by omitting the partition and browser numbers.',
+    },
     {
       name: 'random',
       type: String,
@@ -183,6 +191,14 @@ module.exports = TestCommand.extend({
       );
     }
 
+    if (commandOptions.preserveTestName) {
+      commandOptions.query = addToQuery(
+        commandOptions.query,
+        'preserveTestName',
+        commandOptions.preserveTestName
+      );
+    }
+
     if (commandOptions.filePath) {
       commandOptions.query = addToQuery(
         commandOptions.query,

diff --git a/node-tests/unit/commands/exam-test.js b/node-tests/unit/commands/exam-test.js
@@ -81,6 +81,12 @@ describe('ExamCommand', function () {
       });
     });
 
+    it('should set `preserve-test-name` in the query option', function () {
+      return command.run({ preserveTestName: true }).then(function () {
+        assert.strictEqual(called.testRunOptions.query, 'preserveTestName');
+      });
+    });
+
     it('should set `partition` in the query option with multiple partitions', function () {
       return command.run({ split: 2, partition: [1, 2] }).then(function () {
         assert.strictEqual(

diff --git a/tests/dummy/app/router.js b/tests/dummy/app/router.js
@@ -15,6 +15,7 @@ Router.map(function () {
     this.route('split-parallel');
     this.route('filtering');
     this.route('load-balancing');
+    this.route('preserve-test-name');
 
     this.route('ember-try-and-ci');
     this.route('test-suite-segmentation');

diff --git a/tests/dummy/app/templates/docs.hbs b/tests/dummy/app/templates/docs.hbs
@@ -13,6 +13,7 @@
     {{nav.item "Split Test Parallelization" "docs.split-parallel"}}
     {{nav.item "Filtering" "docs.filtering"}}
     {{nav.item "Test Load Balancing" "docs.load-balancing"}}
+    {{nav.item "Preserve Test Name" "docs.preserve-test-name"}}
 
     {{nav.section "Advanced Configuration"}}
     {{nav.item "Ember Try & CI Integration" "docs.ember-try-and-ci"}}

diff --git a/tests/dummy/app/templates/docs/preserve-test-name.md b/tests/dummy/app/templates/docs/preserve-test-name.md
@@ -0,0 +1,26 @@
+# Preserve Test Name
+
+When using `--split` and/or `--load-balance` the output will look something like:
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=3 --load-balance
+ok 1 Chrome 66.0 - Exam Partition 1 - browser Id 1 - some test
+ok 2 Chrome 66.0 - Exam Partition 1 - browser Id 2 - another test
+ok 3 Chrome 66.0 - Exam Partition 1 - browser Id 3 - some the other test
+```
+However, if you change the amount of parallelization, or randomize accross partitions, the output will change for the same test, which may be an issue if you are tracking test insights over time. 
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=2 --load-balance
+ok 1 Chrome 66.0 - Exam Partition 1 - browser Id 2 - some test
+ok 2 Chrome 66.0 - Exam Partition 1 - browser Id 1 - another test
+ok 3 Chrome 66.0 - Exam Partition 1 - browser Id 2 - some the other test
+```
+You can add `--preserve-test-name` to remove the dynamic segments of the output (partition and browser) to ensure the output test names are always the same.
+
+```bash
+# ember exam --split=2 --partition=1 --parallel=3 --load-balance --preserve-test-name
+ok 1 Chrome 66.0 - some test
+ok 2 Chrome 66.0 - another test
+ok 3 Chrome 66.0 - some the other test
+```
diff --git a/tests/unit/mocha/testem-output-test.js b/tests/unit/mocha/testem-output-test.js
@@ -10,44 +10,94 @@ if (macroCondition(dependencySatisfies('ember-mocha', '*'))) {
   let { expect } = importSync('chai');
 
   describe('Unit | Mocha | patch-testem-output', () => {
-    it('add partition number to test name when `split` is passed', function () {
-      expect(
-        TestemOutput.updateTestName(
-          new Map().set('split', 2),
-          'test_module | test_name'
-        )
-      ).to.equal('Exam Partition 1 - test_module | test_name');
-    });
+    describe('`preserveTestName` is passed', () => {
+      it('does not add partition number to test name when `split` is passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map().set('split', 2).set('preserveTestName', true),
+            'test_module | test_name'
+          )
+        ).to.equal('test_module | test_name');
+      });
 
-    it('add partition number to test name when `split` and `partition` are passed', function () {
-      expect(
-        TestemOutput.updateTestName(
-          new Map().set('split', 2).set('partition', 2),
-          'test_module | test_name'
-        )
-      ).to.equal('Exam Partition 2 - test_module | test_name');
-    });
+      it('does not add partition number to test name when `split` and `partition` are passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map()
+              .set('split', 2)
+              .set('partition', 2)
+              .set('preserveTestName', true),
+            'test_module | test_name'
+          )
+        ).to.equal('test_module | test_name');
+      });
+
+      it('does not add browser number to test name when `loadBalance` and `browser` are passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map()
+              .set('loadBalance', 2)
+              .set('browser', 1)
+              .set('preserveTestName', true),
+            'test_module | test_name'
+          )
+        ).to.equal('test_module | test_name');
+      });
 
-    it('add browser number to test name when `loadBalance` and `browser` are passed', function () {
-      expect(
-        TestemOutput.updateTestName(
-          new Map().set('loadBalance', 2).set('browser', 1),
-          'test_module | test_name'
-        )
-      ).to.equal('Browser Id 1 - test_module | test_name');
+      it('does not add partition number, browser number to test name when `split`, `partition`, `browser`, and `loadBalance` are  passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map()
+              .set('split', 2)
+              .set('partition', 2)
+              .set('browser', 1)
+              .set('loadBalance', 2)
+              .set('preserveTestName', true),
+            'test_module | test_name'
+          )
+        ).to.equal('test_module | test_name');
+      });
     });
+    describe('`preserveTestName` is not passed', () => {
+      it('adds partition number to test name when `split` is passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map().set('split', 2),
+            'test_module | test_name'
+          )
+        ).to.equal('Exam Partition 1 - test_module | test_name');
+      });
+
+      it('adds partition number to test name when `split` and `partition` are passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map().set('split', 2).set('partition', 2),
+            'test_module | test_name'
+          )
+        ).to.equal('Exam Partition 2 - test_module | test_name');
+      });
+
+      it('adds browser number to test name when `loadBalance` and `browser` are passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map().set('loadBalance', 2).set('browser', 1),
+            'test_module | test_name'
+          )
+        ).to.equal('Browser Id 1 - test_module | test_name');
+      });
 
-    it('add partition number, browser number to test name when `split`, `partition`, `browser`, and `loadBalance` are  passed', function () {
-      expect(
-        TestemOutput.updateTestName(
-          new Map()
-            .set('split', 2)
-            .set('partition', 2)
-            .set('browser', 1)
-            .set('loadBalance', 2),
-          'test_module | test_name'
-        )
-      ).to.equal('Exam Partition 2 - Browser Id 1 - test_module | test_name');
+      it('adds partition number, browser number to test name when `split`, `partition`, `browser`, and `loadBalance` are  passed', function () {
+        expect(
+          TestemOutput.updateTestName(
+            new Map()
+              .set('split', 2)
+              .set('partition', 2)
+              .set('browser', 1)
+              .set('loadBalance', 2),
+            'test_module | test_name'
+          )
+        ).to.equal('Exam Partition 2 - Browser Id 1 - test_module | test_name');
+      });
     });
   });
 }