Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Serialize using mapstructure conversion instead of JSON marshalling. #1401

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Serialize using mapstructure conversion instead of JSON marshalling. #1401

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
053c93d
Serialize using mapstructure conversion instead of JSON marshalling.
slewsys Nov 3, 2023
35d607b
Restore ConvertStructToMap to FlattenMetadata.
slewsys Jan 3, 2024
bcd2490
Encode non-string values with ValueToString.
slewsys Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions age/keysource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
)

const (
privateKeySizeLimit = 1 << 24 // 16 MiB
felixfontein marked this conversation as resolved.
Show resolved Hide resolved
// SopsAgeKeyEnv can be set as an environment variable with a string list
// of age keys as value.
SopsAgeKeyEnv = "SOPS_AGE_KEY"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/lib/pq v1.10.9
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/go-wordwrap v1.0.1
github.com/mitchellh/mapstructure v1.5.0
github.com/ory/dockertest/v3 v3.10.0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.9.3
Expand Down Expand Up @@ -105,7 +106,6 @@ require (
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
Expand Down
2 changes: 1 addition & 1 deletion stores/dotenv/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) {
if comment, ok := item.Key.(sops.Comment); ok {
line = fmt.Sprintf("#%s\n", comment.Value)
} else {
value := strings.Replace(item.Value.(string), "\n", "\\n", -1)
value := strings.Replace(stores.ValueToString(item.Value), "\n", "\\n", -1)
line = fmt.Sprintf("%s=%s\n", item.Key, value)
}
buffer.WriteString(line)
Expand Down
16 changes: 1 addition & 15 deletions stores/ini/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"

"strconv"
"strings"

"github.com/getsops/sops/v3"
Expand Down Expand Up @@ -55,7 +54,7 @@ func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) {
lastItem.Comment = comment.Value
}
} else {
lastItem, err = section.NewKey(keyVal.Key.(string), store.valToString(keyVal.Value))
lastItem, err = section.NewKey(keyVal.Key.(string), stores.ValueToString(keyVal.Value))
if err != nil {
return nil, fmt.Errorf("Error encoding key: %s", err)
}
Expand All @@ -77,19 +76,6 @@ func (store Store) stripCommentChar(comment string) string {
return comment
}

func (store Store) valToString(v interface{}) string {
switch v := v.(type) {
case fmt.Stringer:
return v.String()
case float64:
return strconv.FormatFloat(v, 'f', 6, 64)
case bool:
return strconv.FormatBool(v)
default:
return fmt.Sprintf("%s", v)
}
}

func (store Store) iniFromTreeBranches(branches sops.TreeBranches) ([]byte, error) {
return store.encodeTree(branches)
}
Expand Down
175 changes: 129 additions & 46 deletions stores/stores.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ of the purpose of this package is to make it easy to change the SOPS file format
package stores

import (
"reflect"
"strconv"
"time"

"fmt"
Expand All @@ -21,6 +23,7 @@ import (
"github.com/getsops/sops/v3/hcvault"
"github.com/getsops/sops/v3/kms"
"github.com/getsops/sops/v3/pgp"
"github.com/mitchellh/mapstructure"
)

const (
Expand All @@ -42,73 +45,73 @@ type SopsFile struct {
// in order to allow the binary format to stay backwards compatible over time, but at the same time allow the internal
// representation SOPS uses to change over time.
type Metadata struct {
ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty"`
KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty"`
KMSKeys []kmskey `yaml:"kms" json:"kms"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv"`
VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"`
AgeKeys []agekey `yaml:"age" json:"age"`
LastModified string `yaml:"lastmodified" json:"lastmodified"`
MessageAuthenticationCode string `yaml:"mac" json:"mac"`
PGPKeys []pgpkey `yaml:"pgp" json:"pgp"`
UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"`
EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"`
UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty"`
EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"`
MACOnlyEncrypted bool `yaml:"mac_only_encrypted,omitempty" json:"mac_only_encrypted,omitempty"`
Version string `yaml:"version" json:"version"`
ShamirThreshold int `yaml:"shamir_threshold,omitempty" json:"shamir_threshold,omitempty" mapstructure:"shamir_threshold,omitempty"`
KeyGroups []keygroup `yaml:"key_groups,omitempty" json:"key_groups,omitempty" mapstructure:"key_groups,omitempty"`
KMSKeys []kmskey `yaml:"kms" json:"kms" mapstructure:"kms"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms" mapstructure:"gcp_kms"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv" mapstructure:"azure_kv"`
VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault" mapstructure:"hc_vault"`
AgeKeys []agekey `yaml:"age" json:"age" mapstructure:"age"`
LastModified string `yaml:"lastmodified" json:"lastmodified" mapstructure:"lastmodified"`
MessageAuthenticationCode string `yaml:"mac" json:"mac" mapstructure:"mac"`
PGPKeys []pgpkey `yaml:"pgp" json:"pgp" mapstructure:"pgp"`
UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty" mapstructure:"unencrypted_suffix,omitempty"`
EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty" mapstructure:"encrypted_suffix,omitempty"`
UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty" mapstructure:"unencrypted_regex,omitempty"`
EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty" mapstructure:"encrypted_regex,omitempty"`
MACOnlyEncrypted bool `yaml:"mac_only_encrypted,omitempty" json:"mac_only_encrypted,omitempty" mapstructure:"mac_only_encrypted,omitempty"`
Version string `yaml:"version" json:"version" mapstructure:"version"`
}

type keygroup struct {
PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty"`
KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"`
VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"`
AgeKeys []agekey `yaml:"age" json:"age"`
PGPKeys []pgpkey `yaml:"pgp,omitempty" json:"pgp,omitempty" mapstructure:"pgp,omitempty"`
KMSKeys []kmskey `yaml:"kms,omitempty" json:"kms,omitempty" mapstructure:"kms,omitempty"`
GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty" mapstructure:"gcp_kms,omitempty"`
AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty" mapstructure:"azure_kv,omitempty"`
VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault" mapstructure:"hc_vault"`
AgeKeys []agekey `yaml:"age" json:"age" mapstructure:"age"`
}

type pgpkey struct {
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
Fingerprint string `yaml:"fp" json:"fp"`
CreatedAt string `yaml:"created_at" json:"created_at" mapstructure:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
Fingerprint string `yaml:"fp" json:"fp" mapstructure:"fp"`
}

type kmskey struct {
Arn string `yaml:"arn" json:"arn"`
Role string `yaml:"role,omitempty" json:"role,omitempty"`
Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
AwsProfile string `yaml:"aws_profile" json:"aws_profile"`
Arn string `yaml:"arn" json:"arn" mapstructure:"arn"`
Role string `yaml:"role,omitempty" json:"role,omitempty" mapstructure:"role,omitempty"`
Context map[string]*string `yaml:"context,omitempty" json:"context,omitempty" mapstructure:"context,omitempty"`
CreatedAt string `yaml:"created_at" json:"created_at" mapstructure:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
AwsProfile string `yaml:"aws_profile" json:"aws_profile" mapstructure:"aws_profile"`
}

type gcpkmskey struct {
ResourceID string `yaml:"resource_id" json:"resource_id"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
ResourceID string `yaml:"resource_id" json:"resource_id" mapstructure:"resource_id"`
CreatedAt string `yaml:"created_at" json:"created_at" mapstructure:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
}

type vaultkey struct {
VaultAddress string `yaml:"vault_address" json:"vault_address"`
EnginePath string `yaml:"engine_path" json:"engine_path"`
KeyName string `yaml:"key_name" json:"key_name"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
VaultAddress string `yaml:"vault_address" json:"vault_address" mapstructure:"vault_address"`
EnginePath string `yaml:"engine_path" json:"engine_path" mapstructure:"engine_path"`
KeyName string `yaml:"key_name" json:"key_name" mapstructure:"key_name"`
CreatedAt string `yaml:"created_at" json:"created_at" mapstructure:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
}

type azkvkey struct {
VaultURL string `yaml:"vault_url" json:"vault_url"`
Name string `yaml:"name" json:"name"`
Version string `yaml:"version" json:"version"`
CreatedAt string `yaml:"created_at" json:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
VaultURL string `yaml:"vault_url" json:"vault_url" mapstructure:"vault_url"`
Name string `yaml:"name" json:"name" mapstructure:"name"`
Version string `yaml:"version" json:"version" mapstructure:"version"`
CreatedAt string `yaml:"created_at" json:"created_at" mapstructure:"created_at"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
}

type agekey struct {
Recipient string `yaml:"recipient" json:"recipient"`
EncryptedDataKey string `yaml:"enc" json:"enc"`
Recipient string `yaml:"recipient" json:"recipient" mapstructure:"recipient"`
EncryptedDataKey string `yaml:"enc" json:"enc" mapstructure:"enc"`
}

// MetadataFromInternal converts an internal SOPS metadata representation to a representation appropriate for storage
Expand Down Expand Up @@ -521,3 +524,83 @@ func HasSopsTopLevelKey(branch sops.TreeBranch) bool {
}
return false
}

// ConvertStructToMap recursively converts a structure to a map[string]interface{} representation while
// respecting all mapstructure tags on the source structure. This is useful when converting complex structures
// to a map suitable for use with the Flatten function.
//
// Note: this will only emit the public fields of a structure, private fields are ignored entirely.
func ConvertStructToMap(input interface{}) (map[string]interface{}, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function is currently not used.

You need to:

  1. modify FlattenMetadata in flatten.go to use this function;
  2. modify UnflattenMetadata in flatten.go to use mapstructure.WeakDecode;
  3. remove the calls to DecodeNonStrings, since they should be no longer needed.

Probably EncodeNonStrings should be changed as well to traverse the structure and call ValueToString for every non-string value, so that EncodeNonStrings no longer needs to know about the exact metadata structure.

Copy link
Author

@slewsys slewsys Jan 3, 2024 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops! Restored ConvertStructToMap and WeakDecode per original patch.

Removed DecodeNonStrings, but TestDecodeNonStrings fails when:

err := DecodeNonStrings(tt.input)

is replaced with just:

_, err := UnflattenMetadata(tt.input)

So restoring DecodeNonStrings for now...

Presently, EncodeNonStrings considers only two cases: mac_only_encrypted and shamir_threshold. Are you thinking of replacing that with something like:

	for k, v := range m {
		if _, ok := v.(string); !ok {
			m[k] = ValueToString(v)
		}
	}

?

var result map[string]interface{}
err := mapstructure.Decode(input, &result)
if err != nil {
return nil, fmt.Errorf("decode struct: %w", err)
}

// Mapstructure stops when the output interface is satisfied, in our case we need to delve further into
// any collections and ensure that all structures are converted to their map representations.
for k, v := range result {
val := reflect.ValueOf(v)
switch val.Kind() {
case reflect.Array:
case reflect.Slice:
elemType := val.Type().Elem()
// Ignore any elements that are already primitive types
if elemType.Kind() != reflect.Interface &&
elemType.Kind() != reflect.Struct {
continue
}

newList := make([]interface{}, val.Len())
for j := 0; j < val.Len(); j++ {
newVal, err := ConvertStructToMap(val.Index(j).Interface())
if err != nil {
return nil, fmt.Errorf("convert array field to map: %w", err)
}

newList[j] = newVal
}
result[k] = newList
case reflect.Map:
elemType := val.Type().Elem()
// Ignore any elements that are already primitive types
if elemType.Kind() != reflect.Interface &&
elemType.Kind() != reflect.Struct {
continue
}

// Non-string keys
if val.Type().Key().Kind() != reflect.String {
return nil, fmt.Errorf("field '%s' is invalid, only map fields with string keys are supported", k)
}

newMap := map[string]interface{}{}
for _, key := range val.MapKeys() {
newVal, err := ConvertStructToMap(val.MapIndex(key).Interface())
if err != nil {
return nil, fmt.Errorf("convert array field to map: %w", err)
}

newMap[key.String()] = newVal
}
result[k] = newMap
}
}

return result, nil
}

// ValueToString converts the input value to a string representation. This is useful when encoding data to plain
// text formats as is done in the ini and dotenv stores.
func ValueToString(v interface{}) string {
switch v := v.(type) {
case fmt.Stringer:
return v.String()
case float64:
return strconv.FormatFloat(v, 'f', 6, 64)
case bool:
return strconv.FormatBool(v)
default:
return fmt.Sprintf("%v", v)
}
}