RFC: Basic implementation of vimdoc on top of vim-plugin-metadata lib

pyproject.toml

@@ -18,6 +18,9 @@ classifiers = [
 ]
 urls = {Repository = "https://github.com/google/vimdoc"}
 dynamic = ["version"]
+dependencies = [
+  "vim-plugin-metadata >= 1.0.0-rc.0",
+]
 
 [project.optional-dependencies]
 completion = ["shtab"]

vimdoc/module.py

@@ -5,6 +5,8 @@
 import os
 import warnings
 
+import vim_plugin_metadata
+
 import vimdoc
 from vimdoc import error
 from vimdoc import parser
@@ -402,53 +404,34 @@ def Modules(directory):
   # Crawl plugin dir and collect parsed blocks for each file path.
   paths_and_blocks = []
   standalone_paths = []
-  autoloaddir = os.path.join(directory, 'autoload')
-  for (root, dirs, files) in os.walk(directory):
-    # Visit files in a stable order, since the ordering of e.g. the Maktaba
-    # flags below depends upon the order that we visit the files.
-    dirs.sort()
-    files.sort()
-
-    # Prune non-standard top-level dirs like 'test'.
-    if root == directory:
-      dirs[:] = [x for x in dirs if x in DOC_SUBDIRS + ['after']]
-    if root == os.path.join(directory, 'after'):
-      dirs[:] = [x for x in dirs if x in DOC_SUBDIRS]
-    for f in files:
-      filename = os.path.join(root, f)
-      if os.path.splitext(filename)[1] == '.vim':
-        relative_path = os.path.relpath(filename, directory)
-        with io.open(filename, encoding='utf-8') as filehandle:
-          lines = list(filehandle)
-          blocks = list(parser.ParseBlocks(lines, filename))
-          # Define implicit maktaba flags for files that call
-          # maktaba#plugin#Enter. These flags have to be special-cased here
-          # because there aren't necessarily associated doc comment blocks and
-          # the name is computed from the file name.
-          if (not relative_path.startswith('autoload' + os.path.sep)
-              and relative_path != os.path.join('plugin', 'flags.vim')
-              and relative_path != os.path.join('instant', 'flags.vim')):
-            if ContainsMaktabaPluginEnterCall(lines):
-              flagpath = relative_path
-              if flagpath.startswith('after' + os.path.sep):
-                flagpath = os.path.relpath(flagpath, 'after')
-              flagblock = Block(vimdoc.FLAG, is_default=True)
-              name_parts = os.path.splitext(flagpath)[0].split(os.path.sep)
-              flagname = name_parts.pop(0)
-              flagname += ''.join('[' + p + ']' for p in name_parts)
-              flagblock.Local(name=flagname)
-              flagblock.AddLine(
-                  'Configures whether {} should be loaded.'.format(
-                      relative_path))
-              default = 0 if flagname == 'plugin[mappings]' else 1
-              # Use unbulleted list to make sure it's on its own line. Use
-              # backtick to avoid helpfile syntax highlighting.
-              flagblock.AddLine(' - Default: {} `'.format(default))
-              blocks.append(flagblock)
-        paths_and_blocks.append((relative_path, blocks))
-        if filename.startswith(autoloaddir):
-          if blocks and blocks[0].globals.get('standalone'):
-            standalone_paths.append(relative_path)
+  for module, blocks in parser.ParsePluginDir(directory):
+    # Define implicit maktaba flags for files that call
+    # maktaba#plugin#Enter. These flags have to be special-cased here
+    # because there aren't necessarily associated doc comment blocks and
+    # the name is computed from the file name.
+    if (not module.path.parts[0] == 'autoload'
+        and module.path != os.path.join('plugin', 'flags.vim')
+        and module.path != os.path.join('instant', 'flags.vim')):
+      if ContainsMaktabaPluginEnterCall(blocks):
+        flagpath = module.path
+        if flagpath.startswith('after' + os.path.sep):
+          flagpath = os.path.relpath(flagpath, 'after')
+        flagblock = Block(vimdoc.FLAG, is_default=True)
+        name_parts = os.path.splitext(flagpath)[0].split(os.path.sep)
+        flagname = name_parts.pop(0)
+        flagname += ''.join('[' + p + ']' for p in name_parts)
+        flagblock.Local(name=flagname)
+        flagblock.AddLine(
+            'Configures whether {} should be loaded.'.format(
+                module.path))
+        default = 0 if flagname == 'plugin[mappings]' else 1
+        # Use unbulleted list to make sure it's on its own line. Use
+        # backtick to avoid helpfile syntax highlighting.
+        flagblock.AddLine(' - Default: {} `'.format(default))
+        blocks.append(flagblock)
+    paths_and_blocks.append((module.path, blocks))
+    if module.path.parts[0] == 'autoload' and blocks and blocks[0].globals.get('standalone'):
+      standalone_paths.append(module.path)
 
   docdir = os.path.join(directory, 'doc')
   if not os.path.isdir(docdir):
@@ -462,7 +445,7 @@ def Modules(directory):
     if GetMatchingStandalonePath(path, standalone_paths) is not None:
       continue
     namespace = None
-    if path.startswith('autoload' + os.path.sep):
+    if path.parts[0] == 'autoload':
       namespace = GetAutoloadNamespace(os.path.relpath(path, 'autoload'))
     for block in blocks:
       main_module.Merge(block, namespace=namespace)
@@ -506,13 +489,12 @@ def GetMatchingStandalonePath(path, standalones):
   return None
 
 
-def ContainsMaktabaPluginEnterCall(lines):
+def ContainsMaktabaPluginEnterCall(nodes):
   """Returns whether lines of vimscript contain a maktaba#plugin#Enter call.
 
   Args:
-    lines: A sequence of vimscript strings to search.
+    nodes: A sequence of parsed vimscript nodes to search.
   """
-  for _, line in parser.EnumerateStripNewlinesAndJoinContinuations(lines):
-    if not parser.IsComment(line) and 'maktaba#plugin#Enter(' in line:
-      return True
-  return False
+  return any(isinstance(node, vim_plugin_metadata.VimNode.Variable)
+             and 'maktaba#plugin#Enter(' in node.init_value_token
+             for node in nodes)

vimdoc/parser.py

@@ -1,74 +1,11 @@
 """The vimdoc parser."""
+from vim_plugin_metadata import VimNode, VimParser
+
 from vimdoc import codeline
 from vimdoc import docline
-
 from vimdoc import error
 from vimdoc import regex
-
-
-def IsComment(line):
-  return regex.comment_leader.match(line)
-
-
-def IsContinuation(line):
-  return regex.line_continuation.match(line)
-
-
-def StripContinuator(line):
-  assert regex.line_continuation.match(line)
-  return regex.line_continuation.sub('', line)
-
-
-def EnumerateStripNewlinesAndJoinContinuations(lines):
-  """Preprocesses the lines of a vimscript file.
-
-  Enumerates the lines, strips the newlines from the end, and joins the
-  continuations.
-
-  Args:
-    lines: The lines of the file.
-  Yields:
-    Each preprocessed line.
-  """
-  lineno, cached = (None, None)
-  for i, line in enumerate(lines):
-    line = line.rstrip('\n')
-    if IsContinuation(line):
-      if cached is None:
-        raise error.CannotContinue('No preceding line.', i)
-      elif IsComment(cached) and not IsComment(line):
-        raise error.CannotContinue('No comment to continue.', i)
-      else:
-        cached += StripContinuator(line)
-      continue
-    if cached is not None:
-      yield lineno, cached
-    lineno, cached = (i, line)
-  if cached is not None:
-    yield lineno, cached
-
-
-def EnumerateParsedLines(lines):
-  vimdoc_mode = False
-  for i, line in EnumerateStripNewlinesAndJoinContinuations(lines):
-    if not vimdoc_mode:
-      if regex.vimdoc_leader.match(line):
-        vimdoc_mode = True
-        # There's no need to yield the blank line if it's an empty starter line.
-        # For example, in:
-        # ""
-        # " @usage whatever
-        # " description
-        # There's no need to yield the first docline as a blank.
-        if not regex.empty_vimdoc_leader.match(line):
-          # A starter line starts with two comment leaders.
-          # If we strip one of them it's a normal comment line.
-          yield i, ParseCommentLine(regex.comment_leader.sub('', line))
-    elif IsComment(line):
-      yield i, ParseCommentLine(line)
-    else:
-      vimdoc_mode = False
-      yield i, ParseCodeLine(line)
+from vimdoc.block import Block
 
 
 def ParseCodeLine(line):
@@ -119,17 +56,48 @@ def ParseBlockDirective(name, rest):
   raise error.UnrecognizedBlockDirective(name)
 
 
-def ParseBlocks(lines, filename):
-  blocks = []
+def ParseBlocksForNodeDocComment(doc, blocks, selection):
+  if doc is None:
+    return
+  for line in doc.splitlines():
+    yield from ParseCommentLine(f'" {line}').Affect(blocks, selection)
+
+def AffectForVimNode(node, blocks, selection):
+  if isinstance(node, VimNode.StandaloneDocComment):
+    yield from ParseBlocksForNodeDocComment(node.doc, blocks, selection)
+    yield from codeline.Blank().Affect(blocks, selection)
+    return
+  doc = getattr(node, 'doc', None)
+  yield from ParseBlocksForNodeDocComment(doc, blocks, selection)
+  if isinstance(node, VimNode.Function):
+    yield from ParseCodeLine('func{bang} {name}({args}) {modifiers}'.format(
+        name=node.name,
+        args=', '.join(node.args),
+        bang='!' if '!' in node.modifiers else '',
+        modifiers=' '.join(mod for mod in node.modifiers if mod != '!')
+      )).Affect(blocks, selection)
+  elif isinstance(node, VimNode.Command):
+    yield from ParseCodeLine("command {modifiers} {name}".format(name=node.name, modifiers=' '.join(node.modifiers))).Affect(blocks, selection)
+  elif isinstance(node, VimNode.Variable):
+    yield from ParseCodeLine("let {name} = {rhs}".format(
+      name=node.name,
+      rhs=node.init_value_token,
+    )).Affect(blocks, selection)
+  elif isinstance(node, VimNode.Flag):
+    yield from ParseCodeLine("call s:plugin.Flag('{name}', {default_value_token})".format(name=node.name, default_value_token=node.default_value_token)).Affect(blocks, selection)
+
+def ParsePluginModule(module):
+  unclosed_blocks = []
   selection = []
-  lineno = 0
-  try:
-    for lineno, line in EnumerateParsedLines(lines):
-      for block in line.Affect(blocks, selection):
-        yield block.Close()
-    for block in codeline.EndOfFile().Affect(blocks, selection):
-      yield block.Close()
-  except error.ParseError as e:
-    e.lineno = lineno + 1
-    e.filename = filename
-    raise
+  yield from ParseBlocksForNodeDocComment(module.doc, unclosed_blocks, selection)
+  yield from codeline.Blank().Affect(unclosed_blocks, selection)
+
+  for node in module.nodes:
+    yield from AffectForVimNode(node, unclosed_blocks, selection)
+  yield from codeline.EndOfFile().Affect(unclosed_blocks, selection)
+
+def ParsePluginDir(directory):
+  vim_parser = VimParser()
+  for module in vim_parser.parse_plugin_dir(directory).content:
+    module_blocks = [block.Close() for block in ParsePluginModule(module)]
+    yield (module, module_blocks)