Skip to content

Add fixes for complex type schemas, handle inline structs and others #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wirelessr merged 5 commits into from
Mar 11, 2025
Merged

Add fixes for complex type schemas, handle inline structs and others #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3a18adf
Handle inline structs with the json 'inline' tag
zenc0ded Feb 18, 2025
78da3f8
Fix schema generation for complex types.
zenc0ded Feb 18, 2025
bf01b06
Add simple record cache and use it to avoid duplicate type schemas
zenc0ded Feb 19, 2025
a6aaa7d
Cleanup json/bson tag handling and bring bson up to parity.
zenc0ded Feb 23, 2025
30f972f
tidy go modules
zenc0ded Feb 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ module github.com/wirelessr/avroschema

go 1.21.4

require github.com/kamva/mgm/v3 v3.5.0
require (
github.com/kamva/mgm/v3 v3.5.0
github.com/stretchr/testify v1.8.4
go.mongodb.org/mongo-driver v1.8.3
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
Expand All @@ -12,13 +16,10 @@ require (
github.com/klauspost/compress v1.13.6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.0.2 // indirect
github.com/xdg-go/stringprep v1.0.2 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.mongodb.org/mongo-driver v1.8.3 // indirect
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
golang.org/x/text v0.3.5 // indirect
Expand Down
9 changes: 6 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,18 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/kamva/mgm/v3 v3.5.0 h1:/2mNshpqwAC9spdzJZ0VR/UZ/SY/PsNTrMjT111KQjM=
github.com/kamva/mgm/v3 v3.5.0/go.mod h1:F4J1hZnXQMkqL3DZgR7Z7BOuiTqQG/JTic3YzliG4jk=
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand All @@ -22,14 +25,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
Expand Down Expand Up @@ -57,8 +58,10 @@ golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
6 changes: 0 additions & 6 deletions mongo/ext.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,6 @@ func MgmExtension(t reflect.Type) any {
return &avroschema.AvroSchema{Type: "long", LogicalType: "timestamp-millis"}
case "ObjectID": // primitive.ObjectID
return "string"
case "DefaultModel": // mgm.DefaultModel
return []*avroschema.AvroSchema{
{Name: "_id", Type: "string"},
{Name: "created_at", Type: "long", LogicalType: "timestamp-millis"},
{Name: "updated_at", Type: "long", LogicalType: "timestamp-millis"},
}
case "M": // bson.M
return "string"
}
Expand Down
4 changes: 2 additions & 2 deletions mongo/ext_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,15 @@ func TestMgmCommonTypes(t *testing.T) {
"name": "Book",
"type": "record",
"fields": [
{ "name": "_id", "type": "string" },
{ "name": "_id", "type": ["null", "string"] },
{ "name": "created_at", "type": "long", "logicalType": "timestamp-millis" },
{ "name": "updated_at", "type": "long", "logicalType": "timestamp-millis" },
{ "name": "name", "type": "string" },
{ "name": "pages", "type": "int" },
{ "name": "obj_id", "type": "string" },
{ "name": "arrived_at", "type": "long", "logicalType": "timestamp-millis" },
{ "name": "ref_data", "type": "string" },
{ "name": "author", "type": "array", "items": "string" }
{ "name": "author", "type": { "type": "array", "items": "string" }}
]
}`

Expand Down
47 changes: 43 additions & 4 deletions reflect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ type Reflector struct {
Make all fields of Record be backward transitive, i.e., all fields are optional.
*/
BeBackwardTransitive bool
EmitAllFields bool // don't skip struct fields which have no struct tags
SkipTagFieldNames bool // don't use json/bson tag names, even if theyre present
Mapper func(reflect.Type) any
recordTypeCache map[string]reflect.Type
}

/*
Expand Down Expand Up @@ -51,7 +54,10 @@ func (r *Reflector) reflectType(t reflect.Type) any {
if t == timeType {
return &AvroSchema{Type: "long", LogicalType: "timestamp-millis"}
}
return r.handleRecord(t)
rec := r.handleRecord(t)
// cache record result for future references
r.recordTypeCache[t.Name()] = t
return rec
case reflect.Map:
if t.Key().Kind() != reflect.String {
// If the key is not a string, then treat the whole object as a string.
Expand All @@ -76,19 +82,43 @@ func (r *Reflector) handleRecord(t reflect.Type) *AvroSchema {
tokens := strings.Split(name, ".")
name = tokens[len(tokens)-1]

if _, ok := r.recordTypeCache[t.Name()]; ok {
return &AvroSchema{Name: name, Type: t.Name()}
}

ret := &AvroSchema{Name: name, Type: "record"}

for i, n := 0, t.NumField(); i < n; i++ { // handle fields
f := t.Field(i)

jsonTag := f.Tag.Get("json")
jsonFieldName, isOptional := GetNameAndOmit(jsonTag)
jStructTag := parseStructTag(jsonTag)
bsonTag := f.Tag.Get("bson")
bStructTag := parseStructTag(bsonTag)
// for inline structs go and pull the fields and append to this record
if jStructTag.Inline || bStructTag.Inline {
ret.Fields = append(ret.Fields, r.handleRecord(f.Type).Fields...)
continue
}

if jsonFieldName == "" && bsonTag == "" {
// unless emitting all fields, ignore fields with no json/bson tag names
if !r.EmitAllFields && jStructTag.Name == "" && bStructTag.Name == "" {
continue
}
ret.Fields = append(ret.Fields, r.reflectEx(f.Type, isOptional, jsonFieldName)...)
fieldName := f.Name
if !r.SkipTagFieldNames {
// prefer bson tag name in attempt at more compatability with this MgmExtension thing, the mapper for which mimics the bson naming
if bStructTag.Name != "" {
fieldName = bStructTag.Name
} else if jStructTag.Name != "" {
fieldName = jStructTag.Name
}
// otherwise must be emitting all fields and so no other choice than to take the go name
}
// This is likely a backwards compatilbity break with whatever the mgm stuff is, as ObjectID is marked optional in bson, not in json.
// previously bson's optional was never considered here.
isOptional := jStructTag.Optional || bStructTag.Optional
ret.Fields = append(ret.Fields, r.reflectEx(f.Type, isOptional, fieldName)...)
}
return ret
}
Expand Down Expand Up @@ -120,12 +150,21 @@ func (r *Reflector) reflectEx(t reflect.Type, isOpt bool, n string) []*AvroSchem
return nil // FIXME: no error handle
}

// If its one of these complex types then name this separately and embed the type as its own schema
// unions are already handled explicitly above, fixed and enums not yet supported.
if !isOpt && (result.Type == "record" || result.Type == "map" || result.Type == "array") {
return []*AvroSchema{{Name: n, Type: ret}}
}

// the rest is single schema
result.Name = n
return []*AvroSchema{result}
}

func (r *Reflector) ReflectFromType(v any) (string, error) {
// currently everything flows through here so (re)init record cache
r.recordTypeCache = make(map[string]reflect.Type)

t := reflect.TypeOf(v)

if t.Kind() == reflect.Ptr {
Expand Down
Loading