feat: add support for dynamic attributes

Finally added support for dynamic attributes. You must give the people
what they want.

Fixes #669, #628, #638

README.md

@@ -33,12 +33,21 @@ class Card extends Dyngoose.Table {
   @Dyngoose.Attribute.Number()
   public id: number
 
-  @Dyngoose.Attribute.String()
+  // Dyngoose supports inferring the attribute types based on the object types
+  // of your values, however, you can also specify strict attribute types,
+  // which offers more utilities
+  @Dyngoose.Attribute()
   public title: string
 
-  @Dyngoose.Attribute.Number()
+  @Dyngoose.Attribute()
   public number: number
 
+  @Dyngoose.Attribute.String({ trim: true })
+  public description: string
+
+  @Dyngoose.Attribute.StringSet()
+  public owners: Set<string>
+
   @Dyngoose.Attribute.Date({ timeToLive: true })
   public expiresAt: Date
 
@@ -106,18 +115,21 @@ const cards = await Card.primaryKey.query({
   title: ['>=', 'Title']
 })
 
-// you can loop through outputs, which is a native JavaScript array
+// You can loop through outputs, which is a native JavaScript array
 for (const card of cards) {
   console.log(card.id, card.title)
 }
 
-// the output contains additional properties
+// The output contains additional properties
 console.log(`Your query returned ${cards.count} and scanned ${cards.scannedCount} documents`)
 
 // Atomic counters, advanced update expressions
 // Increment or decrement automatically, based on the current value in DynamoDB
 card.set('number', 2, { operator: 'increment' }) // if the current value had been 5, it would now be 7
 card.set('number', 2, { operator: 'decrement' }) // if the current value had been 5, it would now be 3
+
+// Use the add or remove operator on Sets to only partially change an attribute
+card.set('owners', ['some value'], { operator: 'add' })
 ```
 
 ### TS Compiler Setting

docs/Attributes.md

@@ -6,6 +6,7 @@ You can also [define custom types](#custom-attribute-types).
 
 | Dyngoose Attribute | DynamoDB type | JavaScript type | Description |
 |-|-|-|-|
+| `@Dyngoose.Attribute` | `S` | `string` | Converts any JavaScript object into a DynamoDB attribute value. |
 | [`@Dyngoose.Attribute.String`](#dyngooseattributestring) | `S` | `string` | Stores string values. |
 | `@Dyngoose.Attribute.Number` | `N` | `number` or `BigInt` | Stores number values. |
 | `@Dyngoose.Attribute.Boolean` | `BOOL` | `boolean` | Stores boolean values. |
@@ -30,13 +31,42 @@ There are several types of Set attributes, `StringSet`, `NumberSet`, and `Binary
 
 ### Dyngoose Attribute Types
 
+#### Dyngoose.Attribute or Dyngoose.Attribute.Dynamic
+
+> [dynamic.metadata.ts](https://github.com/benhutchins/dyngoose/blob/master/src/metadata/attribute-types/dynamic.metadata.ts)
+
+The `Dynamic` attribute relies on AWS's official [AWS.DynamoDB.Converter](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/Converter.html)
+utility. The additional options available from the converter are exposed on the attribute:
+
+```typescript
+@Dyngoose.Attribute.Dynamic({
+  marshallOptions: {},
+  unmarshallOptions: {},
+})
+```
+
+##### Limitations of Dynamic Attributes
+
+Dynamic attributes have many limitations. When possible, it is recommended you use to explicit attribute types but it is useful to have the flexibility.
+
+1. Arrays are always converted to a `List` (`L`) in DynamoDB.
+
+   When your array contains only strings or numbers, a `Set` is often better as it offers additional query and update operators.
+
+2. Cannot use a Dynamic attribute with an attribute in the table's Primary Key or an Index.
+
+   To build indexes properly, the attribute type must be set and the attribute
+   value must be that defined type consistently. Dyngoose defaults the type for
+   Dynamic attributes to a `String` (`S`) in DynamoDB but this is often
+   incorrect and it'll be better to use a specific attribute type class.
+
 #### Dyngoose.Attribute.String
 
 > [string.metadata.ts](https://github.com/benhutchins/dyngoose/blob/master/src/metadata/attribute-types/string.metadata.ts)
 
 The `String` attribute supports additional settings:
 
-```
+```typescript
 @Dyngoose.Attribute.String({
   // trims the value before saving
   trim: true,
@@ -70,7 +100,7 @@ By default, the `Date` attributes stores values in an [ISO 8601](https://en.wiki
 
 The `Date` attribute supports additional settings:
 
-```
+```typescript
 @Dyngoose.Attribute.Date({
   // store value as a Unix timestamp number value
   unixTmestamp: true,
@@ -97,7 +127,7 @@ The `Date` attribute supports additional settings:
 The `List` attribute supports relies on AWS' marshall utility, to convert any
 JavaScript object into a DynamoDB attribute.
 
-```
+```typescript
 @Dyngoose.Attribute.List({
   marshallOptions: {},
   unmarshallOptions: {},

src/decorator/attribute-types/dynamic.spec.ts

@@ -0,0 +1,34 @@
+import { expect } from 'chai'
+import { TestableTable } from '../../setup-tests.spec'
+
+describe('AttributeType/Dynamic', () => {
+  let record: TestableTable
+
+  beforeEach(() => {
+    record = TestableTable.new()
+  })
+
+  it('should store values based on the object type', () => {
+    expect(record.dynamic).eq(null)
+    record.dynamic = 'some value'
+    expect(record.dynamic).eq('some value')
+    expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ S: 'some value' })
+    record.dynamic = 150
+    expect(record.dynamic).eq(150)
+    expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ N: '150' })
+  })
+
+  it('should support dynamic maps', () => {
+    expect(record.dynamic).eq(null)
+    record.dynamic = { a: 'A', b: 'B' } as any
+    expect(record.dynamic).deep.eq({ a: 'A', b: 'B' })
+    expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ M: { a: { S: 'A' }, b: { S: 'B' } } })
+  })
+
+  it('should support dynamic lists', () => {
+    expect(record.dynamic).eq(null)
+    record.dynamic = [1, 2, 3] as any
+    expect(record.dynamic).deep.eq([1, 2, 3])
+    expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ L: [{ N: '1' }, { N: '2' }, { N: '3' }] })
+  })
+})

src/decorator/attribute-types/dynamic.ts

@@ -0,0 +1,34 @@
+import { type AttributeValue } from '@aws-sdk/client-dynamodb'
+import { type IAttributeType } from '../../interfaces'
+import { type DynamicAttributeValue, type DynamicAttributeMetadata } from '../../metadata/attribute-types/dynamic.metadata'
+import { type Table } from '../../table'
+import { AttributeType } from '../../tables/attribute-type'
+import { marshall, unmarshall } from '@aws-sdk/util-dynamodb'
+import { DynamoAttributeType } from '../../dynamo-attribute-types'
+
+export class DynamicAttributeType extends AttributeType<DynamicAttributeValue, DynamicAttributeMetadata>
+  implements IAttributeType<DynamicAttributeValue> {
+  /**
+   * Dynamic types can be any type, but we set it to string.
+   * This is really only used when creating indexes and the type needs to be specified.
+  */
+  type = DynamoAttributeType.String
+
+  constructor(record: Table, propertyName: string, protected metadata?: DynamicAttributeMetadata) {
+    super(record, propertyName, metadata)
+  }
+
+  toDynamo(value: DynamicAttributeValue): AttributeValue {
+    const marshallOptions = {
+      convertEmptyValues: true,
+      removeUndefinedValues: true,
+      ...this.metadata?.marshallOptions ?? {},
+    }
+
+    return marshall({ value }, marshallOptions).value
+  }
+
+  fromDynamo(attributeValue: AttributeValue): DynamicAttributeValue {
+    return unmarshall({ value: attributeValue }, this.metadata?.unmarshallOptions).value
+  }
+}

src/decorator/attribute-types/index.ts

@@ -5,6 +5,7 @@ import { BinaryAttributeType } from './binary'
 import { BinarySetAttributeType } from './binary-set'
 import { BooleanAttributeType } from './boolean'
 import { DateAttributeType } from './date'
+import { DynamicAttributeType } from './dynamic'
 import { ListAttributeType } from './list'
 import { MapAttributeType } from './map'
 import { NumberAttributeType } from './number'
@@ -18,6 +19,7 @@ interface AttributeTypeMap {
   BinarySet: BinarySetAttributeType
   Boolean: BooleanAttributeType
   Date: DateAttributeType
+  Dynamic: DynamicAttributeType
   List: ListAttributeType
   Number: NumberAttributeType
   NumberSet: NumberSetAttributeType
@@ -31,6 +33,7 @@ interface AttributeMetadataMap {
   BinarySet: Metadata.AttributeType.BinarySet
   Boolean: Metadata.AttributeType.Boolean
   Date: Metadata.AttributeType.Date
+  Dynamic: Metadata.AttributeType.Dynamic
   List: Metadata.AttributeType.List
   Number: Metadata.AttributeType.Number
   NumberSet: Metadata.AttributeType.NumberSet
@@ -44,6 +47,7 @@ const AttributeTypes = {
   BinarySet: BinarySetAttributeType,
   Boolean: BooleanAttributeType,
   Date: DateAttributeType,
+  Dynamic: DynamicAttributeType,
   List: ListAttributeType,
   Number: NumberAttributeType,
   NumberSet: NumberSetAttributeType,
@@ -56,15 +60,15 @@ export interface AttributeDefinition {
   getAttribute: (record: Table, propertyName: string) => any
 }
 
-export function Attribute<T extends keyof AttributeTypeMap>(type: T, metadata?: AttributeMetadataMap[T]): AttributeDefinition {
+export function Attribute<T extends keyof AttributeTypeMap>(type?: T, metadata?: AttributeMetadataMap[T]): AttributeDefinition {
   const define = function (record: Table, propertyName: string): void {
-    const AttributeTypeClass: any = AttributeTypes[type]
+    const AttributeTypeClass: any = AttributeTypes[type ?? 'Dynamic']
     const decorator = new AttributeTypeClass(record, propertyName, metadata)
     decorator.decorate()
   }
 
   define.getAttribute = function (record: Table, propertyName: string): any {
-    const AttributeTypeClass: any = AttributeTypes[type]
+    const AttributeTypeClass: any = AttributeTypes[type ?? 'Dynamic']
     const decorator = new AttributeTypeClass(record, propertyName, metadata)
     return decorator.attribute
   }
@@ -79,6 +83,13 @@ export function Attribute<T extends keyof AttributeTypeMap>(type: T, metadata?:
  */
 Attribute.Any = (options?: Metadata.AttributeType.Any) => Attribute('Any', options)
 
+/**
+ * Converts any JavaScript object into a DynamoDB attribute value.
+ *
+ * Uses AWS.DynamoDB.Converter (marshall and unmarshall).
+ */
+Attribute.Dynamic = (options?: Metadata.AttributeType.Any) => Attribute('Dynamic', options)
+
 Attribute.Binary = (options?: Metadata.AttributeType.Binary) => Attribute('Binary', options)
 Attribute.BinarySet = (options?: Metadata.AttributeType.BinarySet) => Attribute('BinarySet', options)
 

src/metadata/attribute-types/dynamic.metadata.ts

@@ -0,0 +1,9 @@
+import { type marshallOptions, type unmarshallOptions } from '@aws-sdk/util-dynamodb'
+import { type AttributeMetadata } from '../attribute'
+
+export type DynamicAttributeValue = any
+
+export interface DynamicAttributeMetadata extends AttributeMetadata<DynamicAttributeValue> {
+  marshallOptions?: marshallOptions
+  unmarshallOptions?: unmarshallOptions
+}

src/metadata/attribute-types/index.ts

@@ -3,6 +3,7 @@ export type { BinaryAttributeMetadata as Binary } from './binary.metadata'
 export type { BinarySetAttributeMetadata as BinarySet } from './binary-set.metadata'
 export type { BooleanAttributeMetadata as Boolean } from './boolean.metadata'
 export type { DateAttributeMetadata as Date } from './date.metadata'
+export type { DynamicAttributeMetadata as Dynamic } from './dynamic.metadata'
 export type { ListAttributeMetadata as List } from './list.metadata'
 export type { MapAttributeMetadata as Map } from './map.metadata'
 export type { NumberAttributeMetadata as Number } from './number.metadata'

src/setup-tests.spec.ts

@@ -20,6 +20,9 @@ export class TestableTable extends Dyngoose.Table {
   @Dyngoose.Attribute.Number({ default: 1 })
   public id: number
 
+  @Dyngoose.Attribute()
+  public dynamic: number | string
+
   @Dyngoose.Attribute.String()
   public title: string