Skip to content

Commit

Permalink
metadata type generation using reflect: add support for modules calls…
Browse files Browse the repository at this point in the history 
… metadata + create Metadata Generator util (#314)

* metadata type-generation-reflect: automate system calls

* metadata type-generation-reflect: automate balances calls

* add all modules calls metadata

* add support for compactu64 metadata type

* signed extra metadata type generation: adjust logic to use the new MetadataGenerator

* remove left comments

* use generic DecodeCompact; improve metadata generation

* rebase develop

* rebase: develop

* remove cumbersome dereferencing style

* increase coverage

* remove more cumbersome ptr dereferences

* refactor: compact as interface; generate new md for some types

* fix: coverage

* pass mdGenerator to all modules & apis from the runtime

* rebase: dev

* increase coverage

* fix: clear metadata before generation
  • Loading branch information
@nicksinch
nicksinch authored Jan 16, 2024
1 parent bd1ca39 commit d13953a
Show file tree
Hide file tree
Showing 77 changed files with 1,259 additions and 797 deletions.
20 changes: 10 additions & 10 deletions api/metadata/module.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,17 @@ type Module struct {
runtimeApiModules []primitives.RuntimeApiModule
runtimeExtrinsic extrinsic.RuntimeExtrinsic
memUtils utils.WasmMemoryTranslator
generator *primitives.MetadataTypeGenerator
logger log.Logger
}

func New(runtimeExtrinsic extrinsic.RuntimeExtrinsic, runtimeApiModules []primitives.RuntimeApiModule, logger log.Logger) Module {
func New(runtimeExtrinsic extrinsic.RuntimeExtrinsic, runtimeApiModules []primitives.RuntimeApiModule, logger log.Logger, generator *primitives.MetadataTypeGenerator) Module {
return Module{
runtimeApiModules: runtimeApiModules,
runtimeExtrinsic: runtimeExtrinsic,
memUtils: utils.NewMemoryTranslator(),
logger: logger,
generator: generator,
}
}

Expand Down Expand Up @@ -74,16 +76,15 @@ func (m Module) Metadata() int64 {

// TODO: logic is very similar to MetadataAtVersion (for v14). Should be refactored at some point
func (m Module) buildMetadata() primitives.Metadata {
metadataTypesIds := primitives.BuildMetadataTypesIdsMap()

metadataTypes := append(primitiveTypes(), basicTypes()...)

metadataTypes = append(metadataTypes, m.runtimeTypes()...)

types, modules, extrinsic := m.runtimeExtrinsic.Metadata(metadataTypesIds)
modules, extrinsic := m.runtimeExtrinsic.Metadata()

mdTypes := m.generator.GetMetadataTypes()
// append types to all
metadataTypes = append(metadataTypes, types...)
metadataTypes = append(metadataTypes, mdTypes...)

runtimeV14Metadata := primitives.RuntimeMetadataV14{
Types: metadataTypes,
Expand All @@ -110,15 +111,14 @@ func (m Module) MetadataAtVersion(dataPtr int32, dataLen int32) int64 {
m.logger.Critical(err.Error())
}

metadataTypesIds := primitives.BuildMetadataTypesIdsMap()

metadataTypes := append(primitiveTypes(), basicTypes()...)

metadataTypes = append(metadataTypes, m.runtimeTypes()...)

switch version {
case sc.U32(primitives.MetadataVersion14):
types, modules, extrinsicV14 := m.runtimeExtrinsic.Metadata(metadataTypesIds)
modules, extrinsicV14 := m.runtimeExtrinsic.Metadata()
types := m.generator.GetMetadataTypes()
metadataTypes = append(metadataTypes, types...)
metadataV14 := primitives.RuntimeMetadataV14{
Types: metadataTypes,
Expand All @@ -133,7 +133,8 @@ func (m Module) MetadataAtVersion(dataPtr int32, dataLen int32) int64 {
}
return m.memUtils.BytesToOffsetAndSize(optionMd.Bytes())
case sc.U32(primitives.MetadataVersion15):
typesV15, modulesV15, extrinsicV15, outerEnums, custom := m.runtimeExtrinsic.MetadataLatest(metadataTypesIds)
modulesV15, extrinsicV15, outerEnums, custom := m.runtimeExtrinsic.MetadataLatest()
typesV15 := m.generator.GetMetadataTypes()
metadataTypes = append(metadataTypes, typesV15...)
metadataV15 := primitives.RuntimeMetadataV15{
Types: metadataTypes,
Expand Down Expand Up @@ -244,7 +245,6 @@ func basicTypes() sc.Sequence[primitives.MetadataType] {
primitives.NewMetadataType(metadata.TypesSequenceU8, "[]byte", primitives.NewMetadataTypeDefinitionSequence(sc.ToCompact(metadata.PrimitiveTypesU8))),
primitives.NewMetadataType(metadata.TypesSequenceU32, "[]uint32", primitives.NewMetadataTypeDefinitionSequence(sc.ToCompact(metadata.PrimitiveTypesU32))),
primitives.NewMetadataType(metadata.TypesCompactU32, "CompactU32", primitives.NewMetadataTypeDefinitionCompact(sc.ToCompact(metadata.PrimitiveTypesU32))),
primitives.NewMetadataType(metadata.TypesCompactU64, "CompactU64", primitives.NewMetadataTypeDefinitionCompact(sc.ToCompact(metadata.PrimitiveTypesU64))),
primitives.NewMetadataType(metadata.TypesCompactU128, "CompactU128", primitives.NewMetadataTypeDefinitionCompact(sc.ToCompact(metadata.PrimitiveTypesU128))),

primitives.NewMetadataType(metadata.TypesFixedU128, "FixedU128", primitives.NewMetadataTypeDefinitionComposite(
Expand Down
22 changes: 9 additions & 13 deletions api/metadata/module_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ import (
"github.com/stretchr/testify/mock"
)

var (
mdGenerator = primitives.NewMetadataTypeGenerator()
)

var (
dataPtr = int32(0)
dataLen = int32(1)
Expand Down Expand Up @@ -143,9 +147,7 @@ func Test_Module_Item(t *testing.T) {
func Test_Module_Metadata(t *testing.T) {
target := setup()

metadataTypesIds := primitives.BuildMetadataTypesIdsMap()

mockRuntimeExtrinsic.On("Metadata", metadataTypesIds).Return(mdTypes, mdModules14, mdExtrinsic)
mockRuntimeExtrinsic.On("Metadata").Return(mdModules14, mdExtrinsic)

builtMeta := target.buildMetadata()

Expand All @@ -159,7 +161,7 @@ func Test_Module_Metadata(t *testing.T) {

assert.Equal(t, ptrAndSize, result)

mockRuntimeExtrinsic.AssertCalled(t, "Metadata", metadataTypesIds)
mockRuntimeExtrinsic.AssertCalled(t, "Metadata")
mockMemoryUtils.AssertCalled(t, "BytesToOffsetAndSize", bMetadata.Bytes())
}

Expand All @@ -182,11 +184,9 @@ func Test_Module_Metadata_AtVersion_14(t *testing.T) {

version14 := sc.U32(primitives.MetadataVersion14)

metadataTypesIds := primitives.BuildMetadataTypesIdsMap()

mockMemoryUtils.On("GetWasmMemorySlice", dataPtr, dataLen).Return(version14.Bytes())

mockRuntimeExtrinsic.On("Metadata", metadataTypesIds).Return(mdTypes, mdModules14, mdExtrinsic)
mockRuntimeExtrinsic.On("Metadata").Return(mdModules14, mdExtrinsic)

metadataV14 := primitives.RuntimeMetadataV14{
Types: metadataTypes,
Expand All @@ -213,8 +213,6 @@ func Test_Module_Metadata_AtVersion_14(t *testing.T) {
func Test_Module_Metadata_AtVersion_15(t *testing.T) {
target := setup()

metadataTypesIds := primitives.BuildMetadataTypesIdsMap()

metadataTypes := getAllMetadataTypes(&target)

version15 := sc.U32(primitives.MetadataVersion15)
Expand All @@ -231,7 +229,7 @@ func Test_Module_Metadata_AtVersion_15(t *testing.T) {

mockMemoryUtils.On("GetWasmMemorySlice", dataPtr, dataLen).Return(version15.Bytes())

mockRuntimeExtrinsic.On("MetadataLatest", metadataTypesIds).Return(mdTypes, mdModules15, mdExtrinsic15, outerEnums, custom)
mockRuntimeExtrinsic.On("MetadataLatest").Return(mdModules15, mdExtrinsic15, outerEnums, custom)

metadataV15 := primitives.RuntimeMetadataV15{
Types: metadataTypes,
Expand Down Expand Up @@ -296,7 +294,7 @@ func setup() Module {
mockRuntimeExtrinsic = new(mocks.RuntimeExtrinsic)
mockMemoryUtils = new(mocks.MemoryTranslator)

target := New(mockRuntimeExtrinsic, []primitives.RuntimeApiModule{}, log.NewLogger())
target := New(mockRuntimeExtrinsic, []primitives.RuntimeApiModule{}, log.NewLogger(), mdGenerator)
target.memUtils = mockMemoryUtils

return target
Expand All @@ -307,7 +305,5 @@ func getAllMetadataTypes(target *Module) sc.Sequence[primitives.MetadataType] {

metadataTypes = append(metadataTypes, target.runtimeTypes()...)

metadataTypes = append(metadataTypes, mdTypes...)

return metadataTypes
}
Binary file modified build/runtime.wasm
View file
Binary file not shown.
71 changes: 22 additions & 49 deletions execution/extrinsic/runtime_extrinsic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,23 @@ type RuntimeExtrinsic interface {
OnFinalize(n sc.U64) error
OnIdle(n sc.U64, remainingWeight primitives.Weight) primitives.Weight
OffchainWorker(n sc.U64)
Metadata(metadataTypesIds map[string]int) (sc.Sequence[primitives.MetadataType], sc.Sequence[primitives.MetadataModuleV14], primitives.MetadataExtrinsicV14)
MetadataLatest(metadataTypesIds map[string]int) (sc.Sequence[primitives.MetadataType], sc.Sequence[primitives.MetadataModuleV15], primitives.MetadataExtrinsicV15, primitives.OuterEnums, primitives.CustomMetadata)
Metadata() (sc.Sequence[primitives.MetadataModuleV14], primitives.MetadataExtrinsicV14)
MetadataLatest() (sc.Sequence[primitives.MetadataModuleV15], primitives.MetadataExtrinsicV15, primitives.OuterEnums, primitives.CustomMetadata)
}

type runtimeExtrinsic struct {
modules []primitives.Module
extra primitives.SignedExtra
logger log.DebugLogger
modules []primitives.Module
extra primitives.SignedExtra
mdGenerator *primitives.MetadataTypeGenerator
logger log.DebugLogger
}

func New(modules []primitives.Module, extra primitives.SignedExtra, logger log.DebugLogger) RuntimeExtrinsic {
func New(modules []primitives.Module, extra primitives.SignedExtra, mdGenerator *primitives.MetadataTypeGenerator, logger log.DebugLogger) RuntimeExtrinsic {
return runtimeExtrinsic{
modules: modules,
extra: extra,
logger: logger,
modules: modules,
extra: extra,
mdGenerator: mdGenerator,
logger: logger,
}
}

Expand Down Expand Up @@ -193,8 +195,7 @@ func (re runtimeExtrinsic) OffchainWorker(n sc.U64) {
}
}

func (re runtimeExtrinsic) Metadata(metadataTypesIds map[string]int) (sc.Sequence[primitives.MetadataType], sc.Sequence[primitives.MetadataModuleV14], primitives.MetadataExtrinsicV14) {
metadataTypes := sc.Sequence[primitives.MetadataType]{}
func (re runtimeExtrinsic) Metadata() (sc.Sequence[primitives.MetadataModuleV14], primitives.MetadataExtrinsicV14) {
modules := sc.Sequence[primitives.MetadataModuleV14]{}

callVariants := sc.Sequence[sc.Option[primitives.MetadataDefinitionVariant]]{}
Expand All @@ -203,9 +204,8 @@ func (re runtimeExtrinsic) Metadata(metadataTypesIds map[string]int) (sc.Sequenc

// iterate all modules and append their types and modules
for _, module := range re.modules {
mTypes, mModule := module.Metadata()
mModule := module.Metadata()

metadataTypes = append(metadataTypes, mTypes...)
mModuleV14 := mModule.ModuleV14
modules = append(modules, mModuleV14)

Expand All @@ -214,41 +214,28 @@ func (re runtimeExtrinsic) Metadata(metadataTypesIds map[string]int) (sc.Sequenc
errorVariants = append(errorVariants, mModuleV14.ErrorDef)
}

// append runtime event
metadataTypes = append(metadataTypes, re.runtimeEvent(eventVariants))

// get the signed extra types and extensions
signedExtraTypes, signedExtensions := re.extra.Metadata(metadataTypesIds)
// append to signed extra types to all types
metadataTypes = append(metadataTypes, signedExtraTypes...)
signedExtensions := re.extra.Metadata()

// create runtime call type
runtimeCall := re.runtimeCall(callVariants)
// append runtime call to all types
metadataTypes = append(metadataTypes, runtimeCall)

runtimeError := re.runtimeError(errorVariants)

metadataTypes = append(metadataTypes, runtimeError)

// create the unchecked extrinsic type using runtime call id
uncheckedExtrinsicType := createUncheckedExtrinsicType(runtimeCall)

// append it to all types
metadataTypes = append(metadataTypes, uncheckedExtrinsicType)
re.mdGenerator.AppendMetadataTypes(sc.Sequence[primitives.MetadataType]{re.runtimeEvent(eventVariants), runtimeCall, runtimeError, uncheckedExtrinsicType})

// create the metadata extrinsic, which uses the id of the unchecked extrinsic and signed extra extensions
extrinsic := primitives.MetadataExtrinsicV14{
Type: uncheckedExtrinsicType.Id,
Version: types.ExtrinsicFormatVersion,
SignedExtensions: signedExtensions,
}

return metadataTypes, modules, extrinsic
return modules, extrinsic
}

func (re runtimeExtrinsic) MetadataLatest(metadataTypesIds map[string]int) (sc.Sequence[primitives.MetadataType], sc.Sequence[primitives.MetadataModuleV15], primitives.MetadataExtrinsicV15, primitives.OuterEnums, primitives.CustomMetadata) {
metadataTypes := sc.Sequence[primitives.MetadataType]{}
func (re runtimeExtrinsic) MetadataLatest() (sc.Sequence[primitives.MetadataModuleV15], primitives.MetadataExtrinsicV15, primitives.OuterEnums, primitives.CustomMetadata) {
modules := sc.Sequence[primitives.MetadataModuleV15]{}

callVariants := sc.Sequence[sc.Option[primitives.MetadataDefinitionVariant]]{}
Expand All @@ -267,40 +254,26 @@ func (re runtimeExtrinsic) MetadataLatest(metadataTypesIds map[string]int) (sc.S

// iterate all modules and append their types and modules
for _, module := range re.modules {
mTypes, mModule := module.Metadata()
mModule := module.Metadata()

moduleV15 := mModule.ModuleV15

metadataTypes = append(metadataTypes, mTypes...)
modules = append(modules, moduleV15)

callVariants = append(callVariants, moduleV15.CallDef)
eventVariants = append(eventVariants, moduleV15.EventDef)
errorVariants = append(errorVariants, moduleV15.ErrorDef)
}
signedExtensions := re.extra.Metadata()

// append runtime event
metadataTypes = append(metadataTypes, re.runtimeEvent(eventVariants))

// get the signed extra types and extensions
signedExtraTypes, signedExtensions := re.extra.Metadata(metadataTypesIds)
// append to signed extra types to all types
metadataTypes = append(metadataTypes, signedExtraTypes...)

// create runtime call type
runtimeCall := re.runtimeCall(callVariants)
// append runtime call to all types
metadataTypes = append(metadataTypes, runtimeCall)

runtimeError := re.runtimeError(errorVariants)

metadataTypes = append(metadataTypes, runtimeError)

// create the unchecked extrinsic type using runtime call id
uncheckedExtrinsicType := createUncheckedExtrinsicType(runtimeCall)

// append it to all types
metadataTypes = append(metadataTypes, uncheckedExtrinsicType)
// append all metadata types
re.mdGenerator.AppendMetadataTypes(sc.Sequence[primitives.MetadataType]{re.runtimeEvent(eventVariants), runtimeCall, runtimeError, uncheckedExtrinsicType})

extrinsicV15 := primitives.MetadataExtrinsicV15{
Version: types.ExtrinsicFormatVersion,
Expand All @@ -311,7 +284,7 @@ func (re runtimeExtrinsic) MetadataLatest(metadataTypesIds map[string]int) (sc.S
SignedExtensions: signedExtensions,
}

return metadataTypes, modules, extrinsicV15, outerEnums, custom
return modules, extrinsicV15, outerEnums, custom
}

func createUncheckedExtrinsicType(runtimeCall primitives.MetadataType) primitives.MetadataType {
Expand Down
Loading

0 comments on commit d13953a

Please sign in to comment.