Skip to content

Commit bfe5079

Browse files
fix: Emitted ACTIVATE_VERSION now honor the stream aliases configured with stream maps
Signed-off-by: Edgar Ramírez Mondragón <edgarrm358@gmail.com>
1 parent 7a0a2e5 commit bfe5079

36 files changed

Lines changed: 148 additions & 103 deletions

‎singer_sdk/streams/core.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -857,12 +857,16 @@ def _write_state_message(self) -> None:
857857

858858
def _write_activate_version_message(self, full_table_version: int) -> None:
859859
"""Write out an ACTIVATE_VERSION message."""
860-
singer.write_message(
861-
singer.ActivateVersionMessage(
862-
stream=self.name,
863-
version=full_table_version,
860+
for stream_map in self.stream_maps:
861+
if isinstance(stream_map, RemoveRecordTransform):
862+
continue
863+
864+
self._tap.write_message(
865+
singer.ActivateVersionMessage(
866+
stream=stream_map.stream_alias,
867+
version=full_table_version,
868+
)
864869
)
865-
)
866870

867871
def _generate_schema_messages(
868872
self,

‎tests/core/test_mapper.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1028,7 +1028,14 @@ def test_mapped_stream(
10281028
):
10291029
snapshot.snapshot_dir = snapshot_dir.joinpath("mapped_stream")
10301030

1031-
tap = MappedTap(config={"stream_maps": stream_maps, **config}, validate_config=True)
1031+
tap = MappedTap(
1032+
config={
1033+
"emit_activate_version_messages": True,
1034+
"stream_maps": stream_maps,
1035+
**config,
1036+
},
1037+
validate_config=True,
1038+
)
10321039
buf = io.StringIO()
10331040
with redirect_stdout(buf):
10341041
tap.sync_all()
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"aliased_stream","version":1640995200}
3+
{"type":"RECORD","stream":"aliased_stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"aliased_stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"aliased_stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
{"type":"SCHEMA","stream":"aliased_stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2+
{"type":"ACTIVATE_VERSION","stream":"aliased_stream","version":1640995200}
23
{"type":"BATCH","stream":"aliased_stream","encoding":{"format":"jsonl","compression":"gzip"},"manifest":["file:///tmp/stream.json.gz"]}
34
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"aliased.stream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"aliased.stream","version":1640995200}
3+
{"type":"RECORD","stream":"aliased.stream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"aliased.stream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"aliased.stream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"__stream_name__","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"__stream_name__","version":1640995200}
3+
{"type":"RECORD","stream":"__stream_name__","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"__stream_name__","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"__stream_name__","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"aliased","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]},"stream_name":{"type":["string","null"]},"original_stream_name":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"aliased","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"aliased","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"aliased","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"aliased","version":1640995200}
3+
{"type":"RECORD","stream":"aliased","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"aliased","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"aliased","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"aliased","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]},"stream_name":{"type":["string","null"]},"original_stream_name":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"mystream","version":1640995200}
3+
{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","stream_name":"mystream","original_stream_name":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"mystream","schema":{"type":"object","properties":{"email":{"type":["string","null"]}},"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"mystream","version":1640995200}
3+
{"type":"RECORD","stream":"mystream","record":{"email":"ALICE@EXAMPLE.COM"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"mystream","record":{"email":"BOB@EXAMPLE.COM"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"mystream","record":{"email":"CHARLIE@EXAMPLE.COM"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]},"source_table":{"type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
2-
{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
3-
{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
4-
{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"time_extracted":"2022-01-01T00:00:00+00:00"}
2+
{"type":"ACTIVATE_VERSION","stream":"mystream","version":1640995200}
3+
{"type":"RECORD","stream":"mystream","record":{"email":"alice@example.com","count":21,"user":{"id":1,"sub":{"num":1,"custom_obj":"obj-hello"},"some_numbers":[3.14,2.718]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
4+
{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
5+
{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z","source_table":"mystream"},"version":1640995200,"time_extracted":"2022-01-01T00:00:00+00:00"}
56
{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}

0 commit comments

Comments
 (0)