Skip to content

API: constants, annotate member vars, relax API pattern, remove junk #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

API: constants, annotate member vars, relax API pattern, remove junk #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 26 additions & 126 deletions docs/api/api-to-markdown.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@
)
parser.add_argument("--out", "-o", metavar='DIR', default="api", help="Output directory for generated endpoints")
parser.add_argument("--disable-no-args", action='store_true', help="disable NO_ARGS enforcement of `Inputs: none`")
parser.add_argument("--dev", action='store_true', help="generate dev mode docs, which include endpoints marked 'Dev-RPC'")
parser.add_argument("--no-sort", "-S", action='store_true', help="disable sorting endpoints by name (use file order)")
parser.add_argument("--no-group", "-G", action='store_true', help="disable grouping endpoints by category")
parser.add_argument("--no-emdash", "-M", action='store_true', help="disable converting ' -- ' to ' — ' (em-dashes)")
parser.add_argument("--rpc", metavar='URL', default="http://public-na.optf.ngo:22023", help="URL to a running oxend RPC node for live example fetching")
parser.add_argument("filename", nargs="+")
args = parser.parse_args()

Expand All @@ -48,22 +46,15 @@
# "Inputs: none."
# "Outputs: none."
# "Member variable."
# "Constant." (auto-detected for simple const/constexpr values).
# "Inputs:" followed by markdown (typically an unordered list) until the next match from this list.
# "Outputs:" followed by markdown
# "Example input:" followed by a code block (i.e. containing json)
# "Example output:" followed by a code block (i.e. json output)
# "Example-JSON-Fetch" goes and fetches the endpoint (live) with the previous example input as the
# "params" value (or no params if "Inputs: none").
# "Old names: a, b, c"
#
# subject to the following rules:
# - each section must have exactly one Input; if the type inherits NO_ARGS then it *must* be an
# "Inputs: none".
# - each section must have exactly one Output
# - "Example input:" section must be immediately followed by an "Example output"
# - "Example output:" sections are permitted without a preceding example input only if the endpoint
# takes no inputs.
# - 0 or more example pairs are permitted.
# - Old names is permitted only once, if it occurs at all; the given names will be indicated as
# deprecated, old names for the endpoint.
#
Expand All @@ -77,39 +68,32 @@
MD_INPUT_HEADER = f"{hdr}# Parameters"
MD_OUTPUT_HEADER = f"{hdr}# Returns"

MD_EXAMPLES_HEADER = f"{hdr}# Examples"
MD_DECL_HEADER = f"{hdr}# Declaration"
MD_EXAMPLE_IN_HDR = f"{hdr}## Input"
MD_EXAMPLE_OUT_HDR = f"{hdr}## Output"

MD_EX_SINGLE_IN_HDR = f"{hdr}# Example Input"
MD_EX_SINGLE_OUT_HDR = f"{hdr}# Example Output"

MD_NO_INPUT = "This endpoint takes no inputs."

RPC_COMMENT = re.compile(r"^\s*/// ?")
RPC_START = re.compile(r"^API:\s*([\w/:&\\\[\]=,\(\)\<\>]+)(.*)$")
DEV_RPC_START = re.compile(r"^Dev-API:\s*([\w/:]+)(.*)$")
API_COMMENT = re.compile(r"^\s*/// ?")
API_START = re.compile(r"^API:\s*(\w+)/(\w+.*?)\s*$")
IN_NONE = re.compile(r"^Inputs?: *[nN]one\.?$")
IN_SOME = re.compile(r"^Inputs?:\s*$")
MEMBER_VAR = re.compile(r"^Member +[vV]ar(?:iable)?\.?$")
CONSTANT = re.compile(r"^Constant\.?$")
DECL_SOME = re.compile(r"^Declaration?:\s*$")
OUT_SOME = re.compile(r"^Outputs?:\s*$")
EXAMPLE_IN = re.compile(r"^Example [iI]nputs?:\s*$")
EXAMPLE_OUT = re.compile(r"^Example [oO]utputs?:\s*$")
EXAMPLE_JSON_FETCH = re.compile(r"^Example-JSON-Fetch\s*$")
OLD_NAMES = re.compile(r"[Oo]ld [nN]ames?:")
PLAIN_NAME = re.compile(r"\w+")
NO_ARGS = re.compile(r"\bNO_ARGS\b")

CONSTANT_DECLARATION = re.compile(r"^\s*(?:static )?const(?:expr)? (?:\w+(::\w+)* )+= ")

input = fileinput.input(args.filename)
rpc_name = None
api_name = None


def error(msg):
print(
f"\x1b[31;1mERROR\x1b[0m[{input.filename()}:{input.filelineno()}] "
f"while parsing endpoint {rpc_name}:",
f"while parsing endpoint {api_name}:",
file=sys.stderr,
)
if msg and isinstance(msg, list):
Expand All @@ -125,8 +109,6 @@ class Parsing(Enum):
INPUTS = auto()
DECL = auto()
OUTPUTS = auto()
EX_IN = auto()
EX_OUT = auto()
NONE = auto()


Expand All @@ -141,36 +123,29 @@ class Parsing(Enum):

cur_file = input.filename()

line, removed_comment = re.subn(RPC_COMMENT, "", line, count=1)
line, removed_comment = re.subn(API_COMMENT, "", line, count=1)
if not removed_comment:
continue

m = re.search(RPC_START, line)
if not m and args.dev:
m = re.search(DEV_RPC_START, line)
m = re.search(API_START, line)
if not m:
continue
if m and m[2]:
error(f"found trailing garbage after 'API: {m[1]}': {m[2]}")
if m[1].count('/') != 1:
error(f"Found invalid API name: expected 'cat/name', not '{m[1]}'")

cat, rpc_name = m[1].split('/')
cat, api_name = m[1], m[2]
if args.no_group:
cat = ''
description, decl, inputs, outputs = "", "", "", ""
done_desc = False
no_inputs = False
member_var = False
examples = []
cur_ex_in = None
constant = False
old_names = []

mode = Parsing.DESC

while True:
line = input.readline()
line, removed_comment = re.subn(RPC_COMMENT, "", line, count=1)
line, removed_comment = re.subn(API_COMMENT, "", line, count=1)
if not removed_comment:
if not decl:
decl_lines = []
Expand All @@ -186,6 +161,9 @@ class Parsing(Enum):
line = input.readline()

if decl_lines:
if re.search(CONSTANT_DECLARATION, decl_lines[0]):
constant = True

if decl_prefix > 0:
decl_lines = [x[decl_prefix:] for x in decl_lines]
decl = '```cpp\n' + '\n'.join(decl_lines) + '\n```'
Expand All @@ -200,6 +178,9 @@ class Parsing(Enum):
elif re.search(MEMBER_VAR, line):
member_var, no_inputs, mode = True, True, Parsing.DESC

elif re.search(CONSTANT, line):
constant, in_inputs, mode = True, True, Parsing.DESC

elif re.search(DECL_SOME, line):
if inputs:
error("found multiple Syntax:")
Expand All @@ -215,51 +196,6 @@ class Parsing(Enum):
error("found multiple Outputs:")
mode = Parsing.OUTPUTS

elif re.search(EXAMPLE_IN, line):
if cur_ex_in is not None:
error("found multiple input examples without paired output examples")
cur_ex_in = ""
mode = Parsing.EX_IN

elif re.search(EXAMPLE_OUT, line):
if not cur_ex_in and not no_inputs:
error(
"found output example without preceding input example (or 'Inputs: none.')"
)
examples.append([cur_ex_in, ""])
cur_ex_in = None
mode = Parsing.EX_OUT

elif re.search(EXAMPLE_JSON_FETCH, line):
if not cur_ex_in and not no_inputs:
error(
"found output example fetch instruction without preceding input (or 'Inputs: none.')"
)
params = None
if cur_ex_in:
params = cur_ex_in.strip()
if not params.startswith("```json\n"):
error("current example input is not tagged as json for Example-JSON-Fetch")
params = params[8:]
if not params.endswith("\n```"):
error("current example input doesn't look right (expected trailing ```)")
params = params[:-4]
try:
params = json.loads(params)
except Exception as e:
error("failed to parse json example input as json")

result = requests.post(args.rpc + "/json_rpc", json={"jsonrpc": "2.0", "id": "0", "method": rpc_name, "params": params}, timeout=30).json()
if 'error' in result:
error(f"JSON fetched example returned an error: {result['error']}")
elif 'result' not in result:
error(f"JSON fetched example doesn't contain a \"result\" key: {result}")
ex_out = json.dumps(result["result"], indent=2, sort_keys=True)

examples.append([cur_ex_in, f"\n```json\n{ex_out}\n```\n"])
cur_ex_in = None
mode = Parsing.NONE

elif re.search(OLD_NAMES, line):
old_names = [x.strip() for x in line.split(':', 1)[1].split(',')]
if not old_names or not all(re.fullmatch(PLAIN_NAME, n) for n in old_names):
Expand All @@ -281,30 +217,15 @@ class Parsing(Enum):
elif mode == Parsing.OUTPUTS:
outputs += line

elif mode == Parsing.EX_IN:
cur_ex_in += line

elif mode == Parsing.EX_OUT:
examples[-1][1] += line

problems = []
# We hit the end of the commented section
if not description or inputs.isspace():
problems.append("endpoint has no description")
if (not inputs or inputs.isspace()) and not member_var:
problems.append(
"endpoint has no inputs description; perhaps you need to add 'Inputs: none.'?"
)
# if not outputs or outputs.isspace():
# problems.append("endpoint has no outputs description")
if cur_ex_in is not None:
if (not inputs or inputs.isspace()) and not member_var and not constant:
problems.append(
"endpoint has a trailing example input without a following example output"
"endpoint has no inputs description; perhaps you need to add "
"'Inputs: none.', 'Member variable.', or 'Constant.'?"
)
if not no_inputs and any(not x[0] or x[0].isspace() for x in examples):
problems.append("found one or more blank input examples")
if any(not x[1] or x[1].isspace() for x in examples):
problems.append("found one or more blank output examples")

if old_names:
s = 's' if len(old_names) > 1 else ''
Expand All @@ -320,15 +241,15 @@ class Parsing(Enum):
error(problems)

md = f"""
{hdr} `{rpc_name}`
{hdr} `{api_name}`

{description}

{MD_DECL_HEADER}

{decl}
"""
if not member_var:
if not member_var and not constant:
md = md + f"""
{MD_INPUT_HEADER}

Expand All @@ -339,34 +260,13 @@ class Parsing(Enum):
{outputs}
"""

if examples:
if len(examples) > 1:
md += f"\n\n{MD_EXAMPLES_HEADER}\n\n"
for ex in examples:
if ex[0] is not None:
md += f"""
{MD_EXAMPLE_IN_HDR}

{ex[0]}
"""
md += f"""
{MD_EXAMPLE_OUT_HDR}

{ex[1]}
"""

else:
if examples[0][0] is not None:
md += f"\n\n{MD_EX_SINGLE_IN_HDR}\n\n{examples[0][0]}"
md += f"\n\n{MD_EX_SINGLE_OUT_HDR}\n\n{examples[0][1]}"

if not args.no_emdash:
md = md.replace(" -- ", " — ")

if cat in endpoints:
endpoints[cat].append((rpc_name, md))
endpoints[cat].append((api_name, md))
else:
endpoints[cat] = [(rpc_name, md)]
endpoints[cat] = [(api_name, md)]

if not endpoints:
error(f"Found no parseable endpoint descriptions in {cur_file}")
Expand Down
4 changes: 4 additions & 0 deletions include/session/config/base.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1253,6 +1253,8 @@ class ConfigBase : public ConfigSig {

/// API: base/ConfigBase::MULTIPART_MAX_WAIT
///
/// Member variable.
///
/// This value controls how long we will store incomplete multipart messages since the last part
/// of such a message that we received, in the hopes of getting the rest of the parts soon. The
/// default is a week: although long, this allows for extended downtime of a multidevice client
Expand All @@ -1267,6 +1269,8 @@ class ConfigBase : public ConfigSig {

/// API: base/ConfigBase::MULTIPART_MAX_REMEMBER
///
/// Member variable.
///
/// This value controls how long we retain the hashes of *completed* multipart config sets (so
/// that we can know to ignore duplicate message parts of messages we have already processed).
///
Expand Down