From 423cbd0b5dd134f858846ff39a72395ec2b33bf3 Mon Sep 17 00:00:00 2001 From: Benjamin Hutchins Date: Thu, 14 Sep 2023 16:27:15 -0400 Subject: [PATCH] feat: add support for dynamic attributes Finally added support for dynamic attributes. You must give the people what they want. Fixes #669, #628, #638 --- README.md | 20 ++++++++--- docs/Attributes.md | 36 +++++++++++++++++-- src/decorator/attribute-types/dynamic.spec.ts | 34 ++++++++++++++++++ src/decorator/attribute-types/dynamic.ts | 34 ++++++++++++++++++ src/decorator/attribute-types/index.ts | 17 +++++++-- .../attribute-types/dynamic.metadata.ts | 9 +++++ src/metadata/attribute-types/index.ts | 1 + src/setup-tests.spec.ts | 3 ++ 8 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 src/decorator/attribute-types/dynamic.spec.ts create mode 100644 src/decorator/attribute-types/dynamic.ts create mode 100644 src/metadata/attribute-types/dynamic.metadata.ts diff --git a/README.md b/README.md index f765b86e..b23ac80d 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,21 @@ class Card extends Dyngoose.Table { @Dyngoose.Attribute.Number() public id: number - @Dyngoose.Attribute.String() + // Dyngoose supports inferring the attribute types based on the object types + // of your values, however, you can also specify strict attribute types, + // which offers more utilities + @Dyngoose.Attribute() public title: string - @Dyngoose.Attribute.Number() + @Dyngoose.Attribute() public number: number + @Dyngoose.Attribute.String({ trim: true }) + public description: string + + @Dyngoose.Attribute.StringSet() + public owners: Set + @Dyngoose.Attribute.Date({ timeToLive: true }) public expiresAt: Date @@ -106,18 +115,21 @@ const cards = await Card.primaryKey.query({ title: ['>=', 'Title'] }) -// you can loop through outputs, which is a native JavaScript array +// You can loop through outputs, which is a native JavaScript array for (const card of cards) { console.log(card.id, card.title) } -// the output contains additional properties +// The output contains additional properties console.log(`Your query returned ${cards.count} and scanned ${cards.scannedCount} documents`) // Atomic counters, advanced update expressions // Increment or decrement automatically, based on the current value in DynamoDB card.set('number', 2, { operator: 'increment' }) // if the current value had been 5, it would now be 7 card.set('number', 2, { operator: 'decrement' }) // if the current value had been 5, it would now be 3 + +// Use the add or remove operator on Sets to only partially change an attribute +card.set('owners', ['some value'], { operator: 'add' }) ``` ### TS Compiler Setting diff --git a/docs/Attributes.md b/docs/Attributes.md index 0736a663..44e717d9 100644 --- a/docs/Attributes.md +++ b/docs/Attributes.md @@ -6,6 +6,7 @@ You can also [define custom types](#custom-attribute-types). | Dyngoose Attribute | DynamoDB type | JavaScript type | Description | |-|-|-|-| +| `@Dyngoose.Attribute` | `S` | `string` | Converts any JavaScript object into a DynamoDB attribute value. | | [`@Dyngoose.Attribute.String`](#dyngooseattributestring) | `S` | `string` | Stores string values. | | `@Dyngoose.Attribute.Number` | `N` | `number` or `BigInt` | Stores number values. | | `@Dyngoose.Attribute.Boolean` | `BOOL` | `boolean` | Stores boolean values. | @@ -30,13 +31,42 @@ There are several types of Set attributes, `StringSet`, `NumberSet`, and `Binary ### Dyngoose Attribute Types +#### Dyngoose.Attribute or Dyngoose.Attribute.Dynamic + +> [dynamic.metadata.ts](https://github.com/benhutchins/dyngoose/blob/master/src/metadata/attribute-types/dynamic.metadata.ts) + +The `Dynamic` attribute relies on AWS's official [AWS.DynamoDB.Converter](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/Converter.html) +utility. The additional options available from the converter are exposed on the attribute: + +```typescript +@Dyngoose.Attribute.Dynamic({ + marshallOptions: {}, + unmarshallOptions: {}, +}) +``` + +##### Limitations of Dynamic Attributes + +Dynamic attributes have many limitations. When possible, it is recommended you use to explicit attribute types but it is useful to have the flexibility. + +1. Arrays are always converted to a `List` (`L`) in DynamoDB. + + When your array contains only strings or numbers, a `Set` is often better as it offers additional query and update operators. + +2. Cannot use a Dynamic attribute with an attribute in the table's Primary Key or an Index. + + To build indexes properly, the attribute type must be set and the attribute + value must be that defined type consistently. Dyngoose defaults the type for + Dynamic attributes to a `String` (`S`) in DynamoDB but this is often + incorrect and it'll be better to use a specific attribute type class. + #### Dyngoose.Attribute.String > [string.metadata.ts](https://github.com/benhutchins/dyngoose/blob/master/src/metadata/attribute-types/string.metadata.ts) The `String` attribute supports additional settings: -``` +```typescript @Dyngoose.Attribute.String({ // trims the value before saving trim: true, @@ -70,7 +100,7 @@ By default, the `Date` attributes stores values in an [ISO 8601](https://en.wiki The `Date` attribute supports additional settings: -``` +```typescript @Dyngoose.Attribute.Date({ // store value as a Unix timestamp number value unixTmestamp: true, @@ -97,7 +127,7 @@ The `Date` attribute supports additional settings: The `List` attribute supports relies on AWS' marshall utility, to convert any JavaScript object into a DynamoDB attribute. -``` +```typescript @Dyngoose.Attribute.List({ marshallOptions: {}, unmarshallOptions: {}, diff --git a/src/decorator/attribute-types/dynamic.spec.ts b/src/decorator/attribute-types/dynamic.spec.ts new file mode 100644 index 00000000..384d593c --- /dev/null +++ b/src/decorator/attribute-types/dynamic.spec.ts @@ -0,0 +1,34 @@ +import { expect } from 'chai' +import { TestableTable } from '../../setup-tests.spec' + +describe('AttributeType/Dynamic', () => { + let record: TestableTable + + beforeEach(() => { + record = TestableTable.new() + }) + + it('should store values based on the object type', () => { + expect(record.dynamic).eq(null) + record.dynamic = 'some value' + expect(record.dynamic).eq('some value') + expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ S: 'some value' }) + record.dynamic = 150 + expect(record.dynamic).eq(150) + expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ N: '150' }) + }) + + it('should support dynamic maps', () => { + expect(record.dynamic).eq(null) + record.dynamic = { a: 'A', b: 'B' } as any + expect(record.dynamic).deep.eq({ a: 'A', b: 'B' }) + expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ M: { a: { S: 'A' }, b: { S: 'B' } } }) + }) + + it('should support dynamic lists', () => { + expect(record.dynamic).eq(null) + record.dynamic = [1, 2, 3] as any + expect(record.dynamic).deep.eq([1, 2, 3]) + expect(record.getAttributeDynamoValue('dynamic')).to.deep.eq({ L: [{ N: '1' }, { N: '2' }, { N: '3' }] }) + }) +}) diff --git a/src/decorator/attribute-types/dynamic.ts b/src/decorator/attribute-types/dynamic.ts new file mode 100644 index 00000000..4349f9ca --- /dev/null +++ b/src/decorator/attribute-types/dynamic.ts @@ -0,0 +1,34 @@ +import { type AttributeValue } from '@aws-sdk/client-dynamodb' +import { type IAttributeType } from '../../interfaces' +import { type DynamicAttributeValue, type DynamicAttributeMetadata } from '../../metadata/attribute-types/dynamic.metadata' +import { type Table } from '../../table' +import { AttributeType } from '../../tables/attribute-type' +import { marshall, unmarshall } from '@aws-sdk/util-dynamodb' +import { DynamoAttributeType } from '../../dynamo-attribute-types' + +export class DynamicAttributeType extends AttributeType + implements IAttributeType { + /** + * Dynamic types can be any type, but we set it to string. + * This is really only used when creating indexes and the type needs to be specified. + */ + type = DynamoAttributeType.String + + constructor(record: Table, propertyName: string, protected metadata?: DynamicAttributeMetadata) { + super(record, propertyName, metadata) + } + + toDynamo(value: DynamicAttributeValue): AttributeValue { + const marshallOptions = { + convertEmptyValues: true, + removeUndefinedValues: true, + ...this.metadata?.marshallOptions ?? {}, + } + + return marshall({ value }, marshallOptions).value + } + + fromDynamo(attributeValue: AttributeValue): DynamicAttributeValue { + return unmarshall({ value: attributeValue }, this.metadata?.unmarshallOptions).value + } +} diff --git a/src/decorator/attribute-types/index.ts b/src/decorator/attribute-types/index.ts index e41beee6..1e78bf30 100644 --- a/src/decorator/attribute-types/index.ts +++ b/src/decorator/attribute-types/index.ts @@ -5,6 +5,7 @@ import { BinaryAttributeType } from './binary' import { BinarySetAttributeType } from './binary-set' import { BooleanAttributeType } from './boolean' import { DateAttributeType } from './date' +import { DynamicAttributeType } from './dynamic' import { ListAttributeType } from './list' import { MapAttributeType } from './map' import { NumberAttributeType } from './number' @@ -18,6 +19,7 @@ interface AttributeTypeMap { BinarySet: BinarySetAttributeType Boolean: BooleanAttributeType Date: DateAttributeType + Dynamic: DynamicAttributeType List: ListAttributeType Number: NumberAttributeType NumberSet: NumberSetAttributeType @@ -31,6 +33,7 @@ interface AttributeMetadataMap { BinarySet: Metadata.AttributeType.BinarySet Boolean: Metadata.AttributeType.Boolean Date: Metadata.AttributeType.Date + Dynamic: Metadata.AttributeType.Dynamic List: Metadata.AttributeType.List Number: Metadata.AttributeType.Number NumberSet: Metadata.AttributeType.NumberSet @@ -44,6 +47,7 @@ const AttributeTypes = { BinarySet: BinarySetAttributeType, Boolean: BooleanAttributeType, Date: DateAttributeType, + Dynamic: DynamicAttributeType, List: ListAttributeType, Number: NumberAttributeType, NumberSet: NumberSetAttributeType, @@ -56,15 +60,15 @@ export interface AttributeDefinition { getAttribute: (record: Table, propertyName: string) => any } -export function Attribute(type: T, metadata?: AttributeMetadataMap[T]): AttributeDefinition { +export function Attribute(type?: T, metadata?: AttributeMetadataMap[T]): AttributeDefinition { const define = function (record: Table, propertyName: string): void { - const AttributeTypeClass: any = AttributeTypes[type] + const AttributeTypeClass: any = AttributeTypes[type ?? 'Dynamic'] const decorator = new AttributeTypeClass(record, propertyName, metadata) decorator.decorate() } define.getAttribute = function (record: Table, propertyName: string): any { - const AttributeTypeClass: any = AttributeTypes[type] + const AttributeTypeClass: any = AttributeTypes[type ?? 'Dynamic'] const decorator = new AttributeTypeClass(record, propertyName, metadata) return decorator.attribute } @@ -79,6 +83,13 @@ export function Attribute(type: T, metadata?: */ Attribute.Any = (options?: Metadata.AttributeType.Any) => Attribute('Any', options) +/** + * Converts any JavaScript object into a DynamoDB attribute value. + * + * Uses AWS.DynamoDB.Converter (marshall and unmarshall). + */ +Attribute.Dynamic = (options?: Metadata.AttributeType.Any) => Attribute('Dynamic', options) + Attribute.Binary = (options?: Metadata.AttributeType.Binary) => Attribute('Binary', options) Attribute.BinarySet = (options?: Metadata.AttributeType.BinarySet) => Attribute('BinarySet', options) diff --git a/src/metadata/attribute-types/dynamic.metadata.ts b/src/metadata/attribute-types/dynamic.metadata.ts new file mode 100644 index 00000000..90c617da --- /dev/null +++ b/src/metadata/attribute-types/dynamic.metadata.ts @@ -0,0 +1,9 @@ +import { type marshallOptions, type unmarshallOptions } from '@aws-sdk/util-dynamodb' +import { type AttributeMetadata } from '../attribute' + +export type DynamicAttributeValue = any + +export interface DynamicAttributeMetadata extends AttributeMetadata { + marshallOptions?: marshallOptions + unmarshallOptions?: unmarshallOptions +} diff --git a/src/metadata/attribute-types/index.ts b/src/metadata/attribute-types/index.ts index f7332be5..725a8f0d 100644 --- a/src/metadata/attribute-types/index.ts +++ b/src/metadata/attribute-types/index.ts @@ -3,6 +3,7 @@ export type { BinaryAttributeMetadata as Binary } from './binary.metadata' export type { BinarySetAttributeMetadata as BinarySet } from './binary-set.metadata' export type { BooleanAttributeMetadata as Boolean } from './boolean.metadata' export type { DateAttributeMetadata as Date } from './date.metadata' +export type { DynamicAttributeMetadata as Dynamic } from './dynamic.metadata' export type { ListAttributeMetadata as List } from './list.metadata' export type { MapAttributeMetadata as Map } from './map.metadata' export type { NumberAttributeMetadata as Number } from './number.metadata' diff --git a/src/setup-tests.spec.ts b/src/setup-tests.spec.ts index 7870bcf2..3f6a8687 100644 --- a/src/setup-tests.spec.ts +++ b/src/setup-tests.spec.ts @@ -20,6 +20,9 @@ export class TestableTable extends Dyngoose.Table { @Dyngoose.Attribute.Number({ default: 1 }) public id: number + @Dyngoose.Attribute() + public dynamic: number | string + @Dyngoose.Attribute.String() public title: string