Skip to content

Wrap Web.Contents so functions using it aren't excluded from M.pq #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wakinniranye31 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Wrap Web.Contents so functions using it aren't excluded from M.pq #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions M.pq
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,17 @@ let
PAT = "", // Personal Access Token (PAT) for GitHub API https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
QueryHeaders = if PAT <> ""
then
[Authorization = "Bearer " & PAT]
[Authorization = "Bearer " & PAT]
else
[],

// Wrapping Web.Contents in a same-signature function lets it be passed into
// Expression.Evaluate's environment without tripping the "can't use a Dynamic
// Data Source" Formula.Firewall error that occurs when Web.Contents itself is
// passed around as a first-class value.
#"Web.Contents wrapper" = (url as text, optional options as nullable record) as any =>
Web.Contents(url, options),

#"Get Trees all trees" = Json.Document(
Web.Contents(
BaseURL,[
Expand Down Expand Up @@ -70,7 +77,7 @@ let
Text.FromBinary(
Binary.FromText(#"Get functions fx")
),
[
[
Any.Type = Any.Type,
Binary.FromText = Binary.FromText,
Binary.ToText = Binary.ToText,
Expand Down Expand Up @@ -110,6 +117,7 @@ let
DateTimeZone.ZoneMinutes = DateTimeZone.ZoneMinutes,
Day.Friday = Day.Friday,
Day.Monday = Day.Monday,
Documentation.InspiredBy = Documentation.InspiredBy,
Duration.Days = Duration.Days,
Duration.From = Duration.From,
Duration.Hours = Duration.Hours,
Expand Down Expand Up @@ -233,8 +241,12 @@ let
Type.Is = Type.Is,
Value.ReplaceMetadata = Value.ReplaceMetadata,
Value.ReplaceType = Value.ReplaceType,
Value.Type = Value.Type
//,Web.Contents = Web.Contents //Unfortunately adding this function to the M code will create a dynamic error :(
Value.Type = Value.Type,
api.github = api.github,
docs.github = docs.github,
e.g = e.g,
vnd.github = vnd.github,
Web.Contents = #"Web.Contents wrapper"
]
)
in
Expand Down
33 changes: 23 additions & 10 deletions M_Creator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
'Text.ContainsAny',
'Text.ListFromString',
'Text.ReplaceMany',
'Web.Contents', #Unfortunately adding this function to the M code will create a dynamic error :(
'Web.Contents', # Passing Web.Contents itself into Expression.Evaluate's environment triggers
# the "can't use a Dynamic Data Source" Formula.Firewall error. A wrapper function
# (see #"Web.Contents wrapper" below) is mapped in via manual_mappings instead -
# see https://community.powerbi.com/t5/Power-Query/Dynamic-data-sources-aren-t-refreshed-in-the-Power-BI-service/td-p/1563630
'api.openai',
'api.worldbank',
'bibb.pro',
Expand All @@ -60,6 +63,13 @@
'Number.Abs'
]

# Identifiers that can't just be mapped as `Name = Name` in the Expression.Evaluate
# environment record and need a custom expression instead (see #"Web.Contents wrapper"
# in the M_Code template below).
manual_mappings = {
'Web.Contents': '#"Web.Contents wrapper"',
}

M_Code = """

let
Expand All @@ -70,10 +80,17 @@
PAT = "", // Personal Access Token (PAT) for GitHub API https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens
QueryHeaders = if PAT <> ""
then
[Authorization = "Bearer " & PAT]
[Authorization = "Bearer " & PAT]
else
[],

// Wrapping Web.Contents in a same-signature function lets it be passed into
// Expression.Evaluate's environment without tripping the "can't use a Dynamic
// Data Source" Formula.Firewall error that occurs when Web.Contents itself is
// passed around as a first-class value.
#"Web.Contents wrapper" = (url as text, optional options as nullable record) as any =>
Web.Contents(url, options),

#"Get Trees all trees" = Json.Document(
Web.Contents(
BaseURL,[
Expand Down Expand Up @@ -132,9 +149,8 @@
Text.FromBinary(
Binary.FromText(#"Get functions fx")
),
[
[
#TextToReplace
//,Web.Contents = Web.Contents //Unfortunately adding this function to the M code will create a dynamic error :(
]
)
in
Expand Down Expand Up @@ -213,13 +229,10 @@ def process_directory(directory):
unique_functions.sort()

print("Extracted Power Query functions (excluding numbers and specified strings):")
result = ""
for i, func in enumerate(unique_functions):
if i == len(unique_functions) - 1:
result += f" {func} = {func}"
else:
result += f" {func} = {func},\n"
entries = [f"{func} = {func}" for func in unique_functions]
entries += [f"{name} = {expression}" for name, expression in manual_mappings.items()]

result = ",\n".join(f" {entry}" for entry in entries)

M_Code = M_Code.replace("#TextToReplace", result)

Expand Down