Feature: dependent assertions as dependencies (#1720)

* Feature: dependent assertions as dependencies

* Action.yaml support for dependent assertions

* Review updates

* spell check and ActionMap

* dependency duplication check O(n)

* Formating

* includeAssertionsForDependencyMap changed to includeAssertionsForDependency

* review fixes

* Lint fixes

* fix shadow beforeEach call

core/actions/assertion.ts

@@ -56,7 +56,8 @@ export const IAssertionConfigProperties = strictKeysOf<IAssertionConfig>()([
   "name",
   "schema",
   "tags",
-  "type"
+  "type",
+  "dependOnDependencyAssertions"
 ]);
 
 /**
@@ -151,7 +152,9 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
   public dependencies(value: Resolvable | Resolvable[]) {
     const newDependencies = Array.isArray(value) ? value : [value];
     newDependencies.forEach(resolvable => {
-      this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
+      const resolvableTarget = resolvableAsTarget(resolvable);
+      this.session.actionAssertionMap.set(resolvableTarget, this);
+      this.proto.dependencyTargets.push(resolvableTarget);
     });
     return this;
   }

core/actions/index.ts

@@ -23,6 +23,7 @@ export type SqlxConfig = (
 
 export abstract class ActionBuilder<T> {
   public session: Session;
+  public includeAssertionsForDependency: Map<string, boolean> = new Map();
 
   constructor(session?: Session) {
     this.session = session;

core/actions/notebook.ts

@@ -1,9 +1,11 @@
 import { verifyObjectMatchesProto } from "df/common/protos";
 import { ActionBuilder } from "df/core/actions";
+import { Resolvable } from "df/core/common";
 import * as Path from "df/core/path";
 import { Session } from "df/core/session";
 import {
   actionConfigToCompiledGraphTarget,
+  addDependenciesToActionDependencyTargets,
   nativeRequire,
   resolveActionsConfigFilename
 } from "df/core/utils";
@@ -18,6 +20,9 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
   // TODO: make this field private, to enforce proto update logic to happen in this class.
   public proto: dataform.INotebook = dataform.Notebook.create();
 
+  // If true, adds the inline assertions of dependencies as direct dependencies for this action. 
+  public dependOnDependencyAssertions: boolean = false;
+
   constructor(
     session?: Session,
     config?: dataform.ActionConfig.NotebookConfig,
@@ -35,9 +40,8 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
     this.proto.target = this.applySessionToTarget(target, config.filename);
     this.proto.canonicalTarget = this.applySessionCanonicallyToTarget(target);
     this.proto.tags = config.tags;
-    this.proto.dependencyTargets = config.dependencyTargets.map(dependencyTarget =>
-      actionConfigToCompiledGraphTarget(dataform.ActionConfig.Target.create(dependencyTarget))
-    );
+    this.dependOnDependencyAssertions = config.dependOnDependencyAssertions;
+    this.dependencies(config.dependencyTargets);
     this.proto.fileName = config.filename;
     if (config.disabled) {
       this.proto.disabled = config.disabled;
@@ -61,6 +65,15 @@ export class Notebook extends ActionBuilder<dataform.Notebook> {
     return this;
   }
 
+  /**
+   * @hidden
+   */
+  public dependencies(value: Resolvable | Resolvable[]) {
+    const newDependencies = Array.isArray(value) ? value : [value];
+    newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
+    return this;
+  }
+
   /**
    * @hidden
    */

core/actions/operation.ts

@@ -16,13 +16,14 @@ import * as Path from "df/core/path";
 import { Session } from "df/core/session";
 import {
   actionConfigToCompiledGraphTarget,
+  addDependenciesToActionDependencyTargets,
   checkExcessProperties,
   nativeRequire,
   resolvableAsTarget,
   resolveActionsConfigFilename,
   setNameAndTarget,
   strictKeysOf,
-  toResolvable
+  toResolvable,
 } from "df/core/utils";
 import { dataform } from "df/protos/ts";
 
@@ -31,10 +32,10 @@ import { dataform } from "df/protos/ts";
  */
 export interface IOperationConfig
   extends IActionConfig,
-    IDependenciesConfig,
-    IDocumentableConfig,
-    INamedConfig,
-    ITargetableConfig {
+  IDependenciesConfig,
+  IDocumentableConfig,
+  INamedConfig,
+  ITargetableConfig {
   /**
    * Declares that this `operations` action creates a dataset which should be referenceable using the `ref` function.
    *
@@ -59,7 +60,8 @@ export const IIOperationConfigProperties = strictKeysOf<IOperationConfig>()([
   "name",
   "schema",
   "tags",
-  "type"
+  "type",
+  "dependOnDependencyAssertions"
 ]);
 
 /**
@@ -72,6 +74,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
   // Hold a reference to the Session instance.
   public session: Session;
 
+  // If true, adds the inline assertions of dependencies as direct dependencies for this action. 
+  public dependOnDependencyAssertions: boolean = false;
+
   // We delay contextification until the final compile step, so hold these here for now.
   private contextableQueries: Contextable<ICommonContext, string | string[]>;
 
@@ -105,7 +110,8 @@ export class Operation extends ActionBuilder<dataform.Operation> {
       tags: config.tags,
       disabled: config.disabled,
       hasOutput: config.hasOutput,
-      description: config.description
+      description: config.description,
+      dependOnDependencyAssertions: config.dependOnDependencyAssertions
     });
 
     this.queries(nativeRequire(config.filename).query);
@@ -118,6 +124,9 @@ export class Operation extends ActionBuilder<dataform.Operation> {
       IIOperationConfigProperties,
       "operation config"
     );
+    if (config.dependOnDependencyAssertions) {
+      this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
+    }
     if (config.dependencies) {
       this.dependencies(config.dependencies);
     }
@@ -155,9 +164,7 @@ export class Operation extends ActionBuilder<dataform.Operation> {
 
   public dependencies(value: Resolvable | Resolvable[]) {
     const newDependencies = Array.isArray(value) ? value : [value];
-    newDependencies.forEach(resolvable => {
-      this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
-    });
+    newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
     return this;
   }
 
@@ -228,6 +235,11 @@ export class Operation extends ActionBuilder<dataform.Operation> {
     return this;
   }
 
+  public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean) {
+    this.dependOnDependencyAssertions = dependOnDependencyAssertions;
+    return this;
+  }
+
   /**
    * @hidden
    */

core/actions/table.ts

@@ -17,6 +17,7 @@ import * as Path from "df/core/path";
 import { Session } from "df/core/session";
 import {
   actionConfigToCompiledGraphTarget,
+  addDependenciesToActionDependencyTargets,
   checkExcessProperties,
   nativeRequire,
   resolvableAsTarget,
@@ -25,7 +26,7 @@ import {
   strictKeysOf,
   tableTypeStringToEnum,
   toResolvable,
-  validateQueryString
+  validateQueryString,
 } from "df/core/utils";
 import { dataform } from "df/protos/ts";
 
@@ -157,10 +158,10 @@ const ITableAssertionsProperties = () =>
  */
 export interface ITableConfig
   extends IActionConfig,
-    IDependenciesConfig,
-    IDocumentableConfig,
-    INamedConfig,
-    ITargetableConfig {
+  IDependenciesConfig,
+  IDocumentableConfig,
+  INamedConfig,
+  ITargetableConfig {
   /**
    * The type of the dataset. For more information on how this setting works, check out some of the [guides](guides)
    * on publishing different types of datasets with Dataform.
@@ -221,7 +222,8 @@ export const ITableConfigProperties = () =>
     "database",
     "columns",
     "description",
-    "materialized"
+    "materialized",
+    "dependOnDependencyAssertions"
   ]);
 
 /**
@@ -256,6 +258,9 @@ export class Table extends ActionBuilder<dataform.Table> {
   // Hold a reference to the Session instance.
   public session: Session;
 
+  // If true, adds the inline assertions of dependencies as direct dependencies for this action. 
+  public dependOnDependencyAssertions: boolean = false;
+
   // We delay contextification until the final compile step, so hold these here for now.
   public contextableQuery: Contextable<ITableContext, string>;
   private contextableWhere: Contextable<ITableContext, string>;
@@ -340,7 +345,8 @@ export class Table extends ActionBuilder<dataform.Table> {
         tags: config.tags,
         disabled: config.disabled,
         description: config.description,
-        bigquery: bigqueryOptions
+        bigquery: bigqueryOptions,
+        dependOnDependencyAssertions: config.dependOnDependencyAssertions
       });
     }
     if (tableType === "view") {
@@ -370,7 +376,8 @@ export class Table extends ActionBuilder<dataform.Table> {
         materialized: config.materialized,
         tags: config.tags,
         description: config.description,
-        bigquery: bigqueryOptions
+        bigquery: bigqueryOptions,
+        dependOnDependencyAssertions: config.dependOnDependencyAssertions
       });
     }
     if (tableType === "incremental") {
@@ -422,7 +429,8 @@ export class Table extends ActionBuilder<dataform.Table> {
         uniqueKey: config.uniqueKey,
         tags: config.tags,
         description: config.description,
-        bigquery: bigqueryOptions
+        bigquery: bigqueryOptions,
+        dependOnDependencyAssertions: config.dependOnDependencyAssertions
       });
     }
     this.query(nativeRequire(tableTypeConfig.filename).query);
@@ -444,6 +452,9 @@ export class Table extends ActionBuilder<dataform.Table> {
     if (config.type) {
       this.type(config.type);
     }
+    if (config.dependOnDependencyAssertions) {
+      this.setDependOnDependencyAssertions(config.dependOnDependencyAssertions);
+    }
     if (config.dependencies) {
       this.dependencies(config.dependencies);
     }
@@ -552,10 +563,7 @@ export class Table extends ActionBuilder<dataform.Table> {
 
   public dependencies(value: Resolvable | Resolvable[]) {
     const newDependencies = Array.isArray(value) ? value : [value];
-    newDependencies.forEach(resolvable => {
-      this.proto.dependencyTargets.push(resolvableAsTarget(resolvable));
-    });
-
+    newDependencies.forEach(resolvable => addDependenciesToActionDependencyTargets(this, resolvable));
     return this;
   }
 
@@ -676,6 +684,11 @@ export class Table extends ActionBuilder<dataform.Table> {
     return this;
   }
 
+  public setDependOnDependencyAssertions(dependOnDependencyAssertions: boolean) {
+    this.dependOnDependencyAssertions = dependOnDependencyAssertions;
+    return this;
+  }
+
   /**
    * @hidden
    */
@@ -740,7 +753,7 @@ export class Table extends ActionBuilder<dataform.Table> {
  * @hidden
  */
 export class TableContext implements ITableContext {
-  constructor(private table: Table, private isIncremental = false) {}
+  constructor(private table: Table, private isIncremental = false) { }
 
   public config(config: ITableConfig) {
     this.table.config(config);

core/common.ts

@@ -133,6 +133,11 @@ export interface IDependenciesConfig {
    * dependencies, then it should be set to `true`.
    */
   hermetic?: boolean;
+
+  /**
+   * If this flag is set to true, assertions depenedent upon any of the dependencies are added as depenedencies as well.
+   */
+  dependOnDependencyAssertions?: boolean;
 }
 
 /**
@@ -219,6 +224,8 @@ export interface ITarget {
   schema?: string;
 
   name?: string;
+
+  includeDependentAssertions?: boolean;
 }
 
 /**