Skip to content

Commit 0e9f65b

Browse files
jf-techjf-tech
authored
Adding fixed-length samples, README, and wiring into cli/server (#120)
1 parent dcbc0ec commit 0e9f65b

13 files changed

+583
-4
lines changed

‎README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ for trying out schemas and inputs, yours and from sample library, to see how tra
1515

1616
Take a detailed look at samples here:
1717
- [csv examples](extensions/omniv21/samples/csv)
18+
- [fixed-length examples](extensions/omniv21/samples/fixedlength)
1819
- [json examples](extensions/omniv21/samples/json)
1920
- [xml examples](extensions/omniv21/samples/xml).
2021
- [edi examples](extensions/omniv21/samples/edi).
@@ -118,7 +119,8 @@ Take a detailed look at samples here:
118119
- Golang 1.14
119120
120121
## Recent Major Feature Additions/Changes
121-
- Added EDI file format support in omniv2 handler.
122+
- Added fixed-length file format support in omniv21 handler.
123+
- Added EDI file format support in omniv21 handler.
122124
- Major restructure/refactoring
123125
- Upgrade omni schema version to `omni.2.1` due a number of incompatible schema changes:
124126
- `'result_type'` -> `'type'`

‎cli/cmd/serverCmd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ func httpPostTransform(w http.ResponseWriter, r *http.Request) {
154154

155155
var (
156156
sampleDir = "../../extensions/omniv21/samples/"
157-
sampleFormats = []string{"csv", "json", "xml", "edi"}
157+
sampleFormats = []string{"csv", "json", "xml", "fixedlength", "edi"}
158158
sampleInputFilenamePattern = regexp.MustCompile("^([0-9]+[_a-zA-Z0-9]+)\\.input\\.[a-z]+$")
159159
)
160160

‎extensions/omniv21/samples/fixedlength/.snapshots/Test1_Single_Row

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[
2+
{
3+
"date": "2019-01-31T12:34:56-08:00",
4+
"high_temperature_fahrenheit": 50.9,
5+
"latitude": 37.7749,
6+
"longitude": 122.4194,
7+
"low_temperature_fahrenheit": 30.2,
8+
"wind": "North 19.26 mph"
9+
},
10+
{
11+
"date": "2020-07-31T01:23:45-05:00",
12+
"high_temperature_fahrenheit": 102.2,
13+
"latitude": 32.7767,
14+
"longitude": 96.797,
15+
"low_temperature_fahrenheit": 95,
16+
"wind": "South East 19.26 mph"
17+
}
18+
]

‎extensions/omniv21/samples/fixedlength/.snapshots/Test2_Multi_Rows

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[
2+
{
3+
"destination_country": "US",
4+
"events": [
5+
{
6+
"event_date": "2019-08-26T12:47:04-05:00",
7+
"location": {
8+
"city": "HAPPYVALLEY",
9+
"state": "FL",
10+
"zip": "54321"
11+
}
12+
}
13+
],
14+
"guaranteed_delivery_date": "2019-08-27T00:00:00",
15+
"tracking_number": "100000103732"
16+
},
17+
{
18+
"destination_country": "US",
19+
"events": [
20+
{
21+
"event_date": "2019-08-26T12:47:04-05:00",
22+
"location": {
23+
"city": "MAGIC BEACH",
24+
"state": "FL",
25+
"zip": "12345"
26+
}
27+
}
28+
],
29+
"guaranteed_delivery_date": "2019-08-27T00:00:00",
30+
"tracking_number": "W938003272"
31+
}
32+
]

‎extensions/omniv21/samples/fixedlength/.snapshots/Test3_Header_Footer

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
[
2+
{
3+
"carrier": "postnl",
4+
"events": [
5+
{
6+
"event_date": "2019-11-05T04:35:23",
7+
"location": {
8+
"country": "NE",
9+
"zip": "0000FF"
10+
}
11+
}
12+
],
13+
"tracking_number": "3SDMNN0129791",
14+
"weight_in_kg": 19.78
15+
},
16+
{
17+
"carrier": "postnl",
18+
"estimated_delivery_date": "2016-08-25T00:00:00",
19+
"events": [
20+
{
21+
"event_date": "2019-11-05T03:30:00",
22+
"location": {
23+
"city": "'S-HERTOGENBOSCH",
24+
"country": "NL",
25+
"zip": "5211EK"
26+
}
27+
}
28+
],
29+
"tracking_number": "3SDMNN0129552",
30+
"weight_in_kg": 19.78
31+
},
32+
{
33+
"carrier": "postnl",
34+
"estimated_delivery_date": "2016-08-25T00:00:00",
35+
"events": [
36+
{
37+
"event_date": "2019-11-05T03:30:00",
38+
"location": {
39+
"country": "NE",
40+
"zip": "0000FF"
41+
}
42+
}
43+
],
44+
"tracking_number": "3SDMNN0129857",
45+
"weight_in_kg": 1.23
46+
},
47+
{
48+
"carrier": "postnl",
49+
"events": [
50+
{
51+
"event_date": "2019-11-05T03:30:00",
52+
"location": {
53+
"city": "ROTTERDAM",
54+
"country": "NL",
55+
"zip": "3043ME"
56+
}
57+
}
58+
],
59+
"tracking_number": "3SDMNN0129501",
60+
"weight_in_kg": 0
61+
}
62+
]

‎extensions/omniv21/samples/fixedlength/1_single_row.input.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2019/01/31T12:34:56-0800 10.5 30.2 N 33 37.7749 122.4194
2+
DO NOT PROCESS
3+
2020/07/31T01:23:45-0500 39 95 SE 8 32.7767 96.7970

‎extensions/omniv21/samples/fixedlength/1_single_row.schema.json

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"parser_settings": {
3+
"version": "omni.2.1",
4+
"file_format_type": "fixed-length"
5+
},
6+
"file_declaration": {
7+
"envelopes" : [
8+
{
9+
"columns": [
10+
{ "name": "DATE", "start_pos": 1, "length": 24 },
11+
{ "name": "HIGH_TEMP_C", "start_pos": 26, "length": 4 },
12+
{ "name": "LOW_TEMP_F", "start_pos": 31, "length": 4 },
13+
{ "name": "WIND_DIR", "start_pos": 36, "length": 2 },
14+
{ "name": "WIND_SPEED_KMH", "start_pos": 9, "length": 2 },
15+
{ "name": "LAT", "start_pos": 42, "length": 8 },
16+
{ "name": "LONG", "start_pos": 51, "length": 8 }
17+
]
18+
}
19+
]
20+
},
21+
"transform_declarations": {
22+
"FINAL_OUTPUT": { "xpath": ".[DATE != 'DO NOT PROCESS']", "object": {
23+
"date": { "custom_func": {
24+
"name": "dateTimeToRFC3339",
25+
"args": [
26+
{ "xpath": "DATE" },
27+
{ "const": "", "_comment": "input timezone" },
28+
{ "const": "", "_comment": "output timezone" }
29+
]
30+
}},
31+
"high_temperature_fahrenheit": { "xpath": "HIGH_TEMP_C", "template": "template_c_to_f" },
32+
"low_temperature_fahrenheit": {
33+
"custom_func": {
34+
"name": "javascript",
35+
"args": [
36+
{ "const": "Math.floor(low_temp_f * 10) / 10" },
37+
{ "const": "low_temp_f" }, { "xpath": "LOW_TEMP_F", "type": "float" }
38+
]
39+
}
40+
},
41+
"wind": { "custom_func": {
42+
"name": "javascript",
43+
"args": [
44+
{ "const": "win_dir + ' ' + Math.floor(kmh * 0.621371 * 100) / 100 + ' mph'" },
45+
{ "const": "win_dir" }, { "xpath": "WIND_DIR", "template": "wind_acronym_mapping" },
46+
{ "const": "kmh" }, { "xpath": "WIND_SPEED_KMH", "type": "float" }
47+
]
48+
}},
49+
"latitude": { "xpath": "LAT", "type": "float" },
50+
"longitude": { "xpath": "LONG", "type": "float" }
51+
}},
52+
"template_c_to_f": {
53+
"custom_func": {
54+
"name": "javascript",
55+
"args": [
56+
{ "const": "Math.floor((temp_c * 9 / 5 + 32) * 10) / 10" },
57+
{ "const": "temp_c" }, { "xpath": ".", "type": "float" }
58+
]
59+
},
60+
"type": "float"
61+
},
62+
"wind_acronym_mapping": {
63+
"custom_func": {
64+
"name": "javascript",
65+
"args": [
66+
{ "const": "dir=='N'?'North':dir=='NE'?'North East':dir=='E'?'East':dir=='SE'?'South East':dir=='S'?'South':dir=='SW'?'South West':dir=='W'?'West':dir=='NW'?'North West':'Tornado'"},
67+
{ "const": "dir" }, { "xpath": "." }
68+
]
69+
}
70+
}
71+
}
72+
}

‎extensions/omniv21/samples/fixedlength/2_multi_rows.input.txt

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
H001TW 0689311345 030 DEL HMDEPOT 20190826124704 EST 1230004321 W841206858 James Bond 20190827 RHONDA W841206858 128 Bird Ave HAPPYVALLEY FL54321 USELGS 030 1 105.00 THDNR 1 1003164515 DLB000977979 Appt set between 00 00 AND 00 00 on 08 27 19 AG 21200 1600 7029083C 314 Algebra Blvd MEDIAN FL31415 US
2+
H0020689311345 BM 040670500067120
3+
H0020689311345 CN 100000103732
4+
H0020689311345 OA 2400 Highway 155 South
5+
H0020689311345 OC Locust Grove
6+
H0020689311345 ON THE HOME DEPOT 6705
7+
H0020689311345 OO W841206858
8+
H0020689311345 OP 30248
9+
H0020689311345 OST GA
10+
H0020689311345 OT US
11+
H0020689311345 PO 7029083C
12+
H001TW 0689311348 030 DEL HMDEPOT 20190826124704 EST 1230001234 W938003272 Jason Bourne 20190827 RHONDA W938003272 123 S 45ST ST MAGIC BEACH FL12345 USELGS 030 1 1.00 THDNR 1 1003621621 Appt set between 00 00 AND 00 00 on 08 27 19 AG 21030 1430 6816137C 314 Algebra Blvd MEDIAN FL31415 US
13+
H0020689311348 BM 040677700277714
14+
H0020689311348 CN
15+
H0020689311348 OA 2500 Highway 155 South
16+
H0020689311348 OC Locust Grove
17+
H0020689311348 ON THE HOME DEPOT 6777
18+
H0020689311348 OO W938003272
19+
H0020689311348 OP 30248
20+
H0020689311348 OST GA
21+
H0020689311348 OT US
22+
H0020689311348 PO 6816137C

‎extensions/omniv21/samples/fixedlength/2_multi_rows.schema.json

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
{
2+
"parser_settings": {
3+
"version": "omni.2.1",
4+
"file_format_type": "fixed-length"
5+
},
6+
"file_declaration": {
7+
"envelopes" : [
8+
{
9+
"by_rows": 11,
10+
"columns": [
11+
{ "name": "tracking_number_h001", "start_pos": 464, "length": 30, "line_pattern": "^H001" },
12+
{ "name": "destination_country", "start_pos": 607, "length": 2, "line_pattern": "^H001" },
13+
{ "name": "guaranteed_delivery_date", "start_pos": 376, "length": 8, "line_pattern": "^H001" },
14+
{ "name": "event_date", "start_pos": 142, "length": 8, "line_pattern": "^H001" },
15+
{ "name": "event_time", "start_pos": 150, "length": 8, "line_pattern": "^H001" },
16+
{ "name": "event_timezone", "start_pos": 158, "length": 4, "line_pattern": "^H001" },
17+
{ "name": "event_city", "start_pos": 564, "length": 30, "line_pattern": "^H001" },
18+
{ "name": "event_state", "start_pos": 594, "length": 2, "line_pattern": "^H001" },
19+
{ "name": "scan_facility_zip", "start_pos": 596, "length": 11, "line_pattern": "^H001" },
20+
{ "name": "tracking_number_h002_cn", "start_pos": 28, "length": 50, "line_pattern": "^H002.{19}CN" }
21+
]
22+
}
23+
]
24+
},
25+
"transform_declarations": {
26+
"FINAL_OUTPUT": { "object": {
27+
"tracking_number": { "template": "tracking_number_template" },
28+
"destination_country": { "xpath": "destination_country" },
29+
"guaranteed_delivery_date": { "template": "guaranteed_delivery_date_template" },
30+
"events": { "array": [
31+
{ "object": {
32+
"event_date": { "template": "event_date_template" },
33+
"location": { "object": {
34+
"city": { "xpath": "event_city" },
35+
"state": { "xpath": "event_state" },
36+
"zip": { "xpath": "scan_facility_zip" }
37+
}}
38+
}}
39+
]}
40+
}},
41+
"tracking_number_template": { "custom_func": {
42+
"name": "upper",
43+
"args": [
44+
{ "custom_func": {
45+
"name": "coalesce",
46+
"args": [
47+
{ "xpath": "tracking_number_h002_cn" },
48+
{ "xpath": "tracking_number_h001" }
49+
]
50+
}}
51+
]
52+
}},
53+
"guaranteed_delivery_date_template": { "custom_func": {
54+
"name": "dateTimeToRFC3339",
55+
"args": [
56+
{ "xpath": "guaranteed_delivery_date", "_comment": "datetime" },
57+
{ "const": "", "_comment": "input timezone" },
58+
{ "const": "", "_comment": "output timezone" }
59+
]
60+
}},
61+
"event_date_template": { "custom_func": {
62+
"name": "dateTimeToRFC3339",
63+
"args": [
64+
{ "custom_func": {
65+
"name": "concat",
66+
"args": [
67+
{ "xpath": "event_date" },
68+
{ "const": "T" },
69+
{ "xpath": "event_time" }
70+
]
71+
}},
72+
{ "xpath": "event_timezone", "_comment": "input timezone" },
73+
{ "const": "", "_comment": "output timezone" }
74+
]
75+
}}
76+
}
77+
}

0 commit comments

Comments
 (0)