Skip to content

Commit de27564

Browse files
frikkyfrikky
committed
Loads of potential recursion bugs right now. Don't release.
1 parent 48bf3e3 commit de27564

File tree

1 file changed

+216
-11
lines changed

1 file changed

+216
-11
lines changed

‎translate.go‎

Lines changed: 216 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,6 @@ func LiquidTranslate(ctx context.Context, userInput, translatedInput []byte) ([]
225225
log.Printf("[ERROR] Schemaless Liquid: Error parsing and rendering template in LiquidTranslate: %v", err)
226226
}
227227

228-
log.Printf("OUT: %s", out)
229-
230228
return []byte(out), nil
231229
}
232230

@@ -717,9 +715,8 @@ func GetExistingStructure(inputStandard string, shuffleConfig ShuffleConfig) ([]
717715
return byteValue, nil
718716
}
719717

720-
// Recurses to find keys deeper in the thingy
721-
// FIXME: Does NOT fully support loops yet
722-
// Should be able to handle jq/shuffle-json format
718+
// Recurses to find keys deeper based on the standard
719+
// Should be able to handle jq/shuffle-json format (and some liquid)
723720
func recurseFindKey(input map[string]interface{}, key string, depth int) (string, error) {
724721
keys := strings.Split(key, ".")
725722
if len(keys) > 1 {
@@ -736,6 +733,7 @@ func recurseFindKey(input map[string]interface{}, key string, depth int) (string
736733
if v == nil {
737734
return "", nil
738735
} else if val, ok := v.(string); ok {
736+
//log.Printf("STRING RETURN (%s): %#v", k, val)
739737
return val, nil
740738
} else if val, ok := v.(map[string]interface{}); ok {
741739
if b, err := json.MarshalIndent(val, "", "\t"); err != nil {
@@ -792,7 +790,7 @@ func recurseFindKey(input map[string]interface{}, key string, depth int) (string
792790
log.Printf("[ERROR] Schemaless reverse (11): Error marshalling list value: %v", err)
793791
}
794792

795-
// Checks if we are supposed to chekc the list or not
793+
// Checks if we are supposed to check the list or not
796794
if len(parsedKey) > 0 && string(parsedKey[0]) == "#" {
797795
// Trim until first dot (.)
798796
if strings.Contains(parsedKey, ".") {
@@ -806,21 +804,41 @@ func recurseFindKey(input map[string]interface{}, key string, depth int) (string
806804
}
807805
}
808806

807+
// This means we need to return MULTIPLE items
808+
// Not just a single one (:
809+
itemList := []any{}
809810
for _, item := range listValue {
810811
if itemMap, ok := item.(map[string]interface{}); ok {
811812
// Recurse into the item
812813
foundValue, err := recurseFindKey(itemMap, parsedKey, depth + 1)
813814
if err != nil {
814815
if debug {
815-
//log.Printf("[ERROR] Schemaless reverse (9): %v", err)
816+
//log.Printf("[ERROR] Schemaless reverse (9): %v", err)
816817
}
817818
} else {
818-
return foundValue, nil
819+
// FIXME: Returning one item at a time doesn't work
820+
// But returning the whole array means we need to re-construct the same array multiple times over :thinking:
821+
itemList = append(itemList, foundValue)
822+
//return foundValue, nil
819823
}
820824
} else {
821825
log.Printf("[ERROR] Schemaless reverse (10): Item in list is not a map[string]interface{}, but %#v", reflect.TypeOf(item))
826+
itemList = append(itemList, item)
822827
}
823828
}
829+
830+
marshalled, err := json.Marshal(itemList)
831+
if err != nil {
832+
log.Printf("[ERROR] Schemaless reverse (12): Error marshalling item list: %v", err)
833+
return "", err
834+
}
835+
836+
// This is to do some custom parsing that makes
837+
// schemaless["value1", "value2"] into the actual parent
838+
// list. This is required
839+
840+
// If we get them recursed with .#.#
841+
return "schemaless_list"+string(marshalled), nil
824842
} else {
825843
log.Printf("[WARNING] Schemaless reverse (8): Invalid key '%s' (%#v) found in list: %v. Should be something like '#0' or '#.key'. Not checking the list.", parsedKey, keys, string(marshalledMap))
826844
}
@@ -833,6 +851,128 @@ func recurseFindKey(input map[string]interface{}, key string, depth int) (string
833851
return "", errors.New(fmt.Sprintf("Key '%s' not found", key))
834852
}
835853

854+
func setNestedMap(m map[string]interface{}, path string, value interface{}) map[string]interface{} {
855+
keys := strings.Split(path, ".")
856+
if len(keys) == 0 {
857+
return m
858+
}
859+
860+
// Build the nested value to merge
861+
nested := value
862+
for i := len(keys) - 1; i >= 0; i-- {
863+
nested = map[string]interface{}{keys[i]: nested}
864+
}
865+
866+
// Deep merge into the existing map
867+
return deepMerge(copyMap(m), nested.(map[string]interface{}))
868+
}
869+
870+
// Deep copy of a map to avoid mutating the original
871+
func copyMap(m map[string]interface{}) map[string]interface{} {
872+
copy := make(map[string]interface{}, len(m))
873+
for k, v := range m {
874+
if subMap, ok := v.(map[string]interface{}); ok {
875+
copy[k] = copyMap(subMap)
876+
} else {
877+
copy[k] = v
878+
}
879+
}
880+
return copy
881+
}
882+
883+
// Deep merge: dst gets merged with src, with nested maps merged recursively
884+
func deepMerge(dst, src map[string]interface{}) map[string]interface{} {
885+
for k, v := range src {
886+
if vMap, ok := v.(map[string]interface{}); ok {
887+
if dv, exists := dst[k]; exists {
888+
if dvMap, ok := dv.(map[string]interface{}); ok {
889+
dst[k] = deepMerge(dvMap, vMap)
890+
continue
891+
}
892+
}
893+
}
894+
dst[k] = v
895+
}
896+
return dst
897+
}
898+
899+
900+
901+
902+
903+
//func handleMultiListItems(translatedInput map[string]interface{}, jsonKey, translationKey string, loopedValue string) map[string]interface{} {
904+
905+
// This function wouldn't be necessary if other recursion functions
906+
// also have this capability themselves
907+
908+
// translatedInput = the parent object to modify. It is actually the parent of the parents' object.
909+
// jsonKey = the key in the parent object WHICH IS A LIST
910+
// parsedValues = the child values from where we have the list
911+
func handleMultiListItems(translatedInput []interface{}, parentKey string, parsedValues map[string]interface{}) ([]interface{}) {
912+
log.Printf("RECURSE: %s!! Key: %#v", parentKey, parsedValues)
913+
if !strings.Contains(parentKey, ".") && len(translatedInput) == 0 {
914+
translatedInput = append(translatedInput, parsedValues)
915+
}
916+
917+
// Start recursing to find the valid keys
918+
// Checks:
919+
// list -> subsub
920+
// maps -> childkeys
921+
// strings -> build it out.
922+
for childKey, v := range parsedValues {
923+
if val, ok := v.(map[string]interface{}); ok {
924+
log.Printf("Map: %s", childKey)
925+
926+
// By passing in translatedInput we allow child objects to modify the parent?
927+
newKey := fmt.Sprintf("%s.%s", parentKey, childKey)
928+
translatedInput = handleMultiListItems(translatedInput, newKey, val)
929+
930+
} else if _, ok := v.([]interface{}); ok {
931+
log.Printf("List: %s (UNHANDLED)", childKey)
932+
} else if val, ok := v.(string); ok {
933+
log.Printf("string: %s", childKey)
934+
if strings.Contains(val, "schemaless_list[") {
935+
log.Printf("SCHEMALESS LIST FOUND")
936+
937+
foundList := strings.Split(val, "schemaless_list")
938+
if len(foundList) >= 2 {
939+
unmarshalledList := []string{}
940+
err := json.Unmarshal([]byte(foundList[1]), &unmarshalledList)
941+
if err != nil {
942+
log.Printf("[ERROR] Schemaless problem in string unmarshal of %s: err", foundList[1], err)
943+
}
944+
945+
log.Printf("Keys: %d", len(unmarshalledList))
946+
if len(translatedInput) < len(unmarshalledList) {
947+
948+
// Reference Item
949+
firstItem := translatedInput[0]
950+
for cnt, listValue := range unmarshalledList {
951+
if cnt >= len(translatedInput) {
952+
translatedInput = append(translatedInput, firstItem)
953+
}
954+
955+
// Update the translatedInput with the new value
956+
newKey := fmt.Sprintf("%s.%s", parentKey, childKey)
957+
// Remove the first key
958+
newKey = strings.SplitN(newKey, ".", 2)[1]
959+
translatedInput[cnt] = setNestedMap(translatedInput[cnt].(map[string]interface{}), newKey, listValue)
960+
961+
log.Printf("\n\nCNT: %d, NewKey: %s, ListValue: %s\n\n", cnt, newKey, listValue)
962+
}
963+
}
964+
}
965+
}
966+
} else {
967+
log.Printf("OTHER: %s (UNHANDLED)", childKey)
968+
//setNestedMap(translatedInput[cnt].(map[string]interface{}), newKey, listValue)
969+
}
970+
}
971+
972+
log.Printf("TRANSLATEDINPUT: %#v", translatedInput)
973+
return translatedInput
974+
}
975+
836976
func runJsonTranslation(ctx context.Context, inputValue []byte, translation map[string]interface{}) ([]byte, []byte, error) {
837977
//log.Printf("Should translate %s based on %s", string(inputValue), translation)
838978

@@ -850,11 +990,11 @@ func runJsonTranslation(ctx context.Context, inputValue []byte, translation map[
850990
// Creating a new map to store the translated values
851991
translatedInput := make(map[string]interface{})
852992
for translationKey, translationValue := range translation {
993+
log.Printf("[DEBUG] Schemaless: Translating key '%s' with value '%v'. ", translationKey, translationValue)
853994

854995
// Find the field in the parsedInput
855996
found := false
856997
for inputKey, inputValue:= range parsedInput {
857-
_ = inputValue
858998
if inputKey != translationValue {
859999
continue
8601000
}
@@ -875,6 +1015,71 @@ func runJsonTranslation(ctx context.Context, inputValue []byte, translation map[
8751015
}
8761016

8771017
translatedInput[translationKey] = translationValue
1018+
} else if val, ok := translationValue.([]interface{}); ok {
1019+
1020+
newOutput := []interface{}{}
1021+
1022+
// The list part here doesn't really work as this is checking the length of the list in the STANDARD - NOT in the value from the user
1023+
// This makes it so that the append really does... nothing
1024+
for _, v := range val {
1025+
if v, ok := v.(map[string]interface{}); !ok {
1026+
log.Printf("[ERROR] Schemaless: Parsed input value is not a map[string]interface{} for key '%s': %v. Type: %#v", translationKey, v, reflect.TypeOf(v))
1027+
newOutput = append(newOutput, v)
1028+
continue
1029+
}
1030+
1031+
// If the value is a map[string]interface{}, we need to recurse it
1032+
newValue := make(map[string]interface{})
1033+
marshalled, err := json.Marshal(v)
1034+
if err != nil {
1035+
log.Printf("[ERROR] Schemaless: Error marshalling value for key '%s': %v", translationKey, err)
1036+
continue
1037+
}
1038+
1039+
err = json.Unmarshal(marshalled, &newValue)
1040+
if err != nil {
1041+
log.Printf("[ERROR] Schemaless: Error unmarshalling value for key '%s': %v", translationKey, err)
1042+
continue
1043+
}
1044+
1045+
output, _, err := runJsonTranslation(ctx, inputValue, newValue)
1046+
if err != nil {
1047+
log.Printf("[ERROR] Schemaless: Error in runJsonTranslation for key '%s': %v", translationKey, err)
1048+
continue
1049+
}
1050+
1051+
// Marshal the output back to a byte
1052+
var outputParsed map[string]interface{}
1053+
err = json.Unmarshal(output, &outputParsed)
1054+
if err != nil {
1055+
log.Printf("[ERROR] Schemaless: Error in unmarshalling output for key '%s': %v", translationKey, err)
1056+
1057+
translatedInput[translationKey] = translationValue
1058+
continue
1059+
}
1060+
1061+
newOutput = append(newOutput, outputParsed)
1062+
1063+
// Hard to optimise for subkeys -> parent control tho
1064+
if strings.Contains(string(output), "schemaless_list[") {
1065+
log.Printf("WHAT IS THE KEY HERE? %#v. Output: %s", translationKey, output)
1066+
newTranslatedInput := handleMultiListItems(newOutput, translationKey, outputParsed)
1067+
translationValue = newTranslatedInput
1068+
1069+
newOutput = []interface{}{}
1070+
//newOutput = append(newOutput, newTranslatedInput)
1071+
break
1072+
}
1073+
}
1074+
1075+
if len(newOutput) > 0 {
1076+
1077+
translatedInput[translationKey] = newOutput
1078+
} else {
1079+
log.Printf("[ERROR] Schemaless: No output found for key '%s' after translation. Keeping original value.", translationKey)
1080+
translatedInput[translationKey] = translationValue
1081+
}
1082+
8781083
} else if val, ok := translationValue.(map[string]interface{}); ok {
8791084

8801085
// Recurse it with the same function again
@@ -923,7 +1128,7 @@ func runJsonTranslation(ctx context.Context, inputValue []byte, translation map[
9231128
recursed, err := recurseFindKey(parsedInput, key, 0)
9241129
if err != nil {
9251130
if debug {
926-
log.Printf("[DEBUG] Schemaless Reverse problem: Error in recurseFindKey for %#v: %v", key, err)
1131+
log.Printf("[DEBUG] Schemaless Reverse problem: Error in RecurseFindKey for %#v: %v", key, err)
9271132
}
9281133
}
9291134

@@ -1139,7 +1344,7 @@ func Translate(ctx context.Context, inputStandard string, inputValue []byte, inp
11391344
}
11401345

11411346
// Doesn't handle list inputs in json
1142-
startValue := string(inputValue)
1347+
startValue := strings.TrimSpace(string(inputValue))
11431348
if !strings.HasPrefix(startValue, "{") || !strings.HasSuffix(startValue, "}") {
11441349
output, err := YamlConvert(startValue)
11451350
if err != nil {

0 commit comments

Comments
 (0)