diff --git a/00_core.ipynb b/00_core.ipynb index 283be87..8f09f25 100644 --- a/00_core.ipynb +++ b/00_core.ipynb @@ -36,7 +36,7 @@ "from datetime import datetime,timedelta\n", "from pprint import pprint\n", "from time import sleep\n", - "import os" + "import os,fnmatch" ] }, { @@ -1681,6 +1681,887 @@ "api.update_contents('README.md', \"Revert README\", committer=person, author=person, content=readme[:-6]);" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "api = GhApi(token=token)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's implement a function to get all valid files of a repo recursively" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@patch\n", + "def get_repo_files(self:GhApi, owner, repo, branch=\"main\"):\n", + " \"Get all file items of a repo.\"\n", + " tree = self.git.get_tree(owner=owner, repo=repo, tree_sha=branch, recursive=True)\n", + " res = []\n", + " for item in tree['tree']:\n", + " if item['type'] == 'blob': res.append(item) \n", + " return L(res)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(#3) [{'path': '.devcontainer.json', 'mode': '100644', 'type': 'blob', 'sha': '8bfa0e952eb318c5c74acaa26a0016c12e13418e', 'size': 569, 'url': 'https://api.github.com/repos/AnswerDotAI/fastcore/git/blobs/8bfa0e952eb318c5c74acaa26a0016c12e13418e'},{'path': '.gitattributes', 'mode': '100644', 'type': 'blob', 'sha': '753b249880d57c22306cf155601bff986622b1a0', 'size': 26, 'url': 'https://api.github.com/repos/AnswerDotAI/fastcore/git/blobs/753b249880d57c22306cf155601bff986622b1a0'},{'path': '.github/workflows/docs.yml', 'mode': '100644', 'type': 'blob', 'sha': 'cde13ab17f1a9cbc112928d71ecadee93cf30383', 'size': 296, 'url': 'https://api.github.com/repos/AnswerDotAI/fastcore/git/blobs/cde13ab17f1a9cbc112928d71ecadee93cf30383'}]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "owner, repo, branch = \"AnswerDotAI\", \"fastcore\", \"main\"\n", + "repo_files = api.get_repo_files(owner,repo); repo_files[:3]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It would be useful to add filter options to further filter these files. We can use [fnmatch](https://docs.python.org/3/library/fnmatch.html) to add Unix shell-style wildcard based filtering which is simple yet pretty flexible." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _find_matches(path, pats):\n", + " \"Returns matched patterns\"\n", + " matches = []\n", + " for p in listify(pats):\n", + " if fnmatch.fnmatch(path,p): matches.append(p)\n", + " return matches" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['*.md', 'README.md']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "_find_matches('README.md', ['*.py', '*test_*', '*/test*/*', '*.md', 'README.md'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _include(path, include, exclude):\n", + " \"Prioritize non-star matches, if both include and exclude star expr then pick longer.\"\n", + " include_matches = [\"*\"] if include is None else _find_matches(path, include)\n", + " exclude_matches = [] if exclude is None else _find_matches(path, exclude)\n", + " if include_matches and exclude_matches:\n", + " include_star = [m for m in include_matches if \"*\" in m]\n", + " exclude_star = [m for m in exclude_matches if \"*\" in m]\n", + " if include_star and exclude_star: return len(include_star) > len(exclude_star)\n", + " if include_star: return False\n", + " if exclude_star: return True \n", + " if include_matches: return True\n", + " if exclude_matches: return False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exclude all .md files expect for README.md" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert _include('README.md', ['README.md'], ['*.md'])\n", + "assert not _include('CONTRIBUTING.md', ['README.md'], ['*.md'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Include all .py files except for tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert not _include('examples/test_fastcore2.py', ['*.py'], ['*test_*', '*/test*/*'])\n", + "assert not _include('examples/tests/some_test.py', ['*.py'], ['*test_*', '*/tests/*'])\n", + "assert not _include('examples/test/some_test.py', ['*.py'], ['*test_*', '*/test/*'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert _include('cool/module.py', ['*.py'], ['setup.py'])\n", + "assert not _include('cool/_modidx', ['*.py'], ['*/_modidx'])\n", + "assert not _include('setup.py', ['*.py'], ['setup.py'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_repo_files = ['README.md', 'CONTRIBUTING.md', 'dir/MARKDOWN.md', 'tests/file.py', \n", + " 'module/file.py', 'module/app/file.py', 'nbs/00.ipynb', 'file2.py',\n", + " '.gitignore', 'module/.dotfile', '_hidden.py', 'module/_hidden.py']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here is an example where we filter to include all python files except for the ones under tests directory, include all notebooks, exclude all md files except for README.md, and all files starting with an underscore. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['README.md',\n", + " 'module/file.py',\n", + " 'module/app/file.py',\n", + " 'nbs/00.ipynb',\n", + " 'file2.py']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inc,exc = ['README.md', '*.py', '*.ipynb'], ['*.md', 'tests/*.py', '_*', '*/_*']\n", + "[fn for fn in test_repo_files if _include(fn,inc,exc)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's exclude files starting with `test_` and `setup.py` too." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['*.md', 'tests/*.py', '_*', '*/_*', '*test_*.py', '*/*test*.py', 'setup.py']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exc += ['*test_*.py', '*/*test*.py', 'setup.py']; exc" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The list of files that are kept based on the filtering logic:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(39,\n", + " ['README.md',\n", + " 'fastcore/all.py',\n", + " 'fastcore/ansi.py',\n", + " 'fastcore/basics.py',\n", + " 'fastcore/dispatch.py',\n", + " 'fastcore/docments.py',\n", + " 'fastcore/docscrape.py',\n", + " 'fastcore/foundation.py',\n", + " 'fastcore/imghdr.py',\n", + " 'fastcore/imports.py',\n", + " 'fastcore/meta.py',\n", + " 'fastcore/nb_imports.py',\n", + " 'fastcore/net.py',\n", + " 'fastcore/parallel.py',\n", + " 'fastcore/py2pyi.py',\n", + " 'fastcore/script.py',\n", + " 'fastcore/shutil.py',\n", + " 'fastcore/style.py',\n", + " 'fastcore/transform.py',\n", + " 'fastcore/utils.py',\n", + " 'fastcore/xdg.py',\n", + " 'fastcore/xml.py',\n", + " 'fastcore/xtras.py',\n", + " 'nbs/000_tour.ipynb',\n", + " 'nbs/00_test.ipynb',\n", + " 'nbs/01_basics.ipynb',\n", + " 'nbs/02_foundation.ipynb',\n", + " 'nbs/03_xtras.ipynb',\n", + " 'nbs/03a_parallel.ipynb',\n", + " 'nbs/03b_net.ipynb',\n", + " 'nbs/04_docments.ipynb',\n", + " 'nbs/05_meta.ipynb',\n", + " 'nbs/06_script.ipynb',\n", + " 'nbs/07_xdg.ipynb',\n", + " 'nbs/08_style.ipynb',\n", + " 'nbs/09_xml.ipynb',\n", + " 'nbs/10_py2pyi.ipynb',\n", + " 'nbs/11_external.ipynb',\n", + " 'nbs/index.ipynb'])" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo_files_filtered = repo_files.filter(lambda o: _include(o.path, inc, exc))\n", + "len(repo_files_filtered), list(repo_files_filtered.map(lambda o: o.path))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below we can see the files that got filtered out:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['.devcontainer.json',\n", + " '.gitattributes',\n", + " '.github/workflows/docs.yml',\n", + " '.github/workflows/main.yml',\n", + " '.gitignore',\n", + " 'CHANGELOG.md',\n", + " 'CODE_OF_CONDUCT.md',\n", + " 'CONTRIBUTING.md',\n", + " 'LICENSE',\n", + " 'MANIFEST.in',\n", + " 'docker-compose.yml',\n", + " 'examples/ansi.css',\n", + " 'examples/test_fastcore.py',\n", + " 'examples/test_fastcore2.py',\n", + " 'fastcore/__init__.py',\n", + " 'fastcore/_modidx.py',\n", + " 'fastcore/_nbdev.py',\n", + " 'fastcore/test.py',\n", + " 'images/att_00000.png',\n", + " 'images/att_00001.png',\n", + " 'images/att_00002.png',\n", + " 'nbs/.gitattributes',\n", + " 'nbs/.gitignore',\n", + " 'nbs/.nojekyll',\n", + " 'nbs/CNAME',\n", + " 'nbs/_parallel_win.ipynb',\n", + " 'nbs/_quarto.yml',\n", + " 'nbs/fastcore',\n", + " 'nbs/files/test.txt.bz2',\n", + " 'nbs/images/att_00000.png',\n", + " 'nbs/images/att_00005.png',\n", + " 'nbs/images/att_00006.png',\n", + " 'nbs/images/att_00007.png',\n", + " 'nbs/images/mnist3.png',\n", + " 'nbs/images/puppy.jpg',\n", + " 'nbs/llms-ctx-full.txt',\n", + " 'nbs/llms-ctx.txt',\n", + " 'nbs/llms.txt',\n", + " 'nbs/nbdev.yml',\n", + " 'nbs/parallel_test.py',\n", + " 'nbs/styles.css',\n", + " 'nbs/test_py2pyi.py',\n", + " 'nbs/test_py2pyi.pyi',\n", + " 'pyproject.toml',\n", + " 'settings.ini',\n", + " 'setup.py']" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "list(repo_files.filter(lambda o: o.path not in repo_files_filtered.attrgot('path')).attrgot('path'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "# Welcome to fastcore\n", + "\n", + "\n", + "\n", + "\n", + "Python is a powerful, dynamic language. Rather than bake everything into\n", + "the language, it lets the programmer customize it to make it work for\n", + "them. `fastcore` uses this flexibility to add to Python features\n", + "inspired by other languages we’ve loved, mixins from Ruby, and currying,\n", + "binding, and more from Haskell. It also adds some “missing features” and\n", + "clean up some rough edges in the Python standard library, such as\n", + "simplifying parallel processing, and bringing ideas from NumPy over to\n", + "Python’s `list` type.\n", + "\n", + "## Getting started\n", + "\n", + "To install fastcore run: `conda install fastcore -c fastai` (if you use\n", + "Anaconda, which we recommend) or `pip install fastcore`. For an\n", + "[editable\n", + "install](https://stackoverflow.com/questions/35064426/when-would-the-e-editable-option-be-useful-with-pip-install),\n", + "clone this repo and run: `pip install -e \".[dev]\"`. fastcore is tested\n", + "to work on Ubuntu, macOS and Windows (versions tested are those shown\n", + "with the `-latest` suffix\n", + "[here](https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources)).\n", + "\n", + "`fastcore` contains many features, including:\n", + "\n", + "- `fastcore.test`: Simple testing functions\n", + "- `fastcore.foundation`: Mixins, delegation, composition, and more\n", + "- `fastcore.xtras`: Utility functions to help with functional-style\n", + " programming, parallel processing, and more\n", + "\n", + "To get started, we recommend you read through [the fastcore\n", + "tour](https://fastcore.fast.ai/tour.html).\n", + "\n", + "## Contributing\n", + "\n", + "After you clone this repository, please run `nbdev_install_hooks` in\n", + "your terminal. This sets up git hooks, which clean up the notebooks to\n", + "remove the extraneous stuff stored in the notebooks (e.g. which cells\n", + "you ran) which causes unnecessary merge conflicts.\n", + "\n", + "To run the tests in parallel, launch `nbdev_test`.\n", + "\n", + "Before submitting a PR, check that the local library and notebooks\n", + "match.\n", + "\n", + "- If you made a change to the notebooks in one of the exported cells,\n", + " you can export it to the library with `nbdev_prepare`.\n", + "- If you made a change to the library, you can export it back to the\n", + " notebooks with `nbdev_update`.\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from IPython.display import Markdown\n", + "item = repo_files_filtered[0]\n", + "content = api.repos.get_content(owner, repo, item['path'])\n", + "content['content_decoded'] = base64.b64decode(content.content).decode('utf-8')\n", + "Markdown(content.content_decoded)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's update `get_repo_files` with the filtering mechanism we've implemented above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@patch\n", + "def get_repo_files(self:GhApi, owner, repo, branch=\"main\", inc=None, exc=None):\n", + " \"Get all file items of a repo.\"\n", + " tree = self.git.get_tree(owner=owner, repo=repo, tree_sha=branch, recursive=True)\n", + " res = L()\n", + " for item in tree['tree']:\n", + " if item['type'] == 'blob': res.append(item) \n", + " return res.filter(lambda o: _include(o.path,inc,exc))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(#39) ['README.md','fastcore/all.py','fastcore/ansi.py','fastcore/basics.py','fastcore/dispatch.py','fastcore/docments.py','fastcore/docscrape.py','fastcore/foundation.py','fastcore/imghdr.py','fastcore/imports.py','fastcore/meta.py','fastcore/nb_imports.py','fastcore/net.py','fastcore/parallel.py','fastcore/py2pyi.py','fastcore/script.py','fastcore/shutil.py','fastcore/style.py','fastcore/transform.py','fastcore/utils.py'...]" + ] + }, + "execution_count": null, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "repo_files = api.get_repo_files(owner, repo, inc=inc, exc=exc); repo_files.attrgot(\"path\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "@patch\n", + "def get_file_content(self:GhApi, path, owner, repo, branch=\"main\"):\n", + " o = self.repos.get_content(owner, repo, path, ref=branch)\n", + " o['content_decoded'] = base64.b64decode(o.content).decode('utf-8')\n", + " return o" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://github.com/AnswerDotAI/fastcore/blob/main/README.md\n", + "# Welcome to fastcore\n", + "\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "o = api.get_file_content(repo_files[0].path, owner, repo)\n", + "_head = \"\\n\".join(o.content_decoded.split(\"\\n\")[:5])\n", + "print(f\"{o.html_url}\\n{_head}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://github.com/AnswerDotAI/fastcore/blob/main/README.md\n", + "# Welcome to fastcore\n", + "\n", + "\n", + "\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/all.py\n", + "from .imports import *\n", + "from .foundation import *\n", + "from .utils import *\n", + "from .parallel import *\n", + "from .net import *\n" + ] + } + ], + "source": [ + "contents = parallel(api.get_file_content, repo_files[:2].attrgot(\"path\"), owner=owner, repo=repo)\n", + "for o in contents:\n", + " _head = \"\\n\".join(o.content_decoded.split(\"\\n\")[:5])\n", + " print(f\"{o.html_url}\\n{_head}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#|export\n", + "@patch\n", + "@delegates(GhApi.get_repo_files)\n", + "def get_repo_contents(self:GhApi, owner, repo, **kwargs):\n", + " repo_files = self.get_repo_files(owner, repo, **kwargs)\n", + " for s in ('inc','exc',): kwargs.pop(s)\n", + " return parallel(self.get_file_content, repo_files.attrgot(\"path\"), owner=owner, repo=repo, **kwargs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inc,exc = ['*.md', \"*.py\"],['*/_*.py', '*test*.py', '*/*test*.py', 'setup.py']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "contents = api.get_repo_contents(owner,repo,branch=\"main\",inc=inc, exc=exc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://github.com/AnswerDotAI/fastcore/blob/main/CHANGELOG.md\n", + "# Release notes\n", + "\n", + "\n", + "\n", + "## 1.8.1\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/CODE_OF_CONDUCT.md\n", + "# Contributor Covenant Code of Conduct\n", + "\n", + "## Our Pledge\n", + "\n", + "In the interest of fostering an open and welcoming environment, we as\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/CONTRIBUTING.md\n", + "# How to contribute\n", + "\n", + "## How to get started\n", + "\n", + "Clone the `fastcore` repository.\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/README.md\n", + "# Welcome to fastcore\n", + "\n", + "\n", + "\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/all.py\n", + "from .imports import *\n", + "from .foundation import *\n", + "from .utils import *\n", + "from .parallel import *\n", + "from .net import *\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/ansi.py\n", + "\"Filters for processing ANSI colors.\"\n", + "\n", + "# Copyright (c) IPython Development Team.\n", + "# Modifications by Jeremy Howard.\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/basics.py\n", + "\"\"\"Basic functionality used in the fastai library\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_basics.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/dispatch.py\n", + "def __getattr__(name):\n", + " raise ImportError(\n", + " f\"Could not import '{name}' from fastcore.dispatch - this module has been moved to the fasttransform package.\\n\"\n", + " \"To migrate your code, please see the migration guide at: https://answerdotai.github.io/fasttransform/fastcore_migration_guide.html\"\n", + " )\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/docments.py\n", + "\"\"\"Document parameters using comments.\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/04_docments.ipynb.\n", + "\n", + "# %% ../nbs/04_docments.ipynb 2\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/docscrape.py\n", + "\"Parse numpy-style docstrings\"\n", + "\n", + "\"\"\"\n", + "Based on code from numpy, which is:\n", + "Copyright (c) 2005-2022, NumPy Developers.\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/foundation.py\n", + "\"\"\"The `L` class and helpers for it\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_foundation.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/imghdr.py\n", + "\"\"\"Recognize image file formats based on their first few bytes.\"\"\"\n", + "\n", + "from os import PathLike\n", + "import warnings\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/imports.py\n", + "import sys,os,re,typing,itertools,operator,functools,math,warnings,functools,io,enum\n", + "\n", + "from operator import itemgetter,attrgetter\n", + "from warnings import warn\n", + "from typing import Iterable,Generator,Sequence,Iterator,List,Set,Dict,Union,Optional,Tuple\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/meta.py\n", + "\"\"\"Metaclasses\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/05_meta.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/nb_imports.py\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import numbers,tempfile,pickle,random,inspect,shutil\n", + "\n", + "from PIL import Image\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/net.py\n", + "\"\"\"Network, HTTP, and URL functions\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03b_net.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/parallel.py\n", + "\"\"\"Threading and multiprocessing functions\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03a_parallel.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/py2pyi.py\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/10_py2pyi.ipynb.\n", + "\n", + "# %% auto 0\n", + "__all__ = ['functypes', 'imp_mod', 'has_deco', 'sig2str', 'ast_args', 'create_pyi', 'py2pyi', 'replace_wildcards']\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/script.py\n", + "\"\"\"A fast way to turn your python function into a script.\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/06_script.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/shutil.py\n", + "from functools import wraps\n", + "import shutil\n", + "\n", + "__all__ = ['copymode', 'copystat', 'copy', 'copy2', 'move', 'copytree', 'rmtree', 'disk_usage', 'chown', 'rmtree']\n", + "\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/style.py\n", + "\"\"\"Fast styling for friendly CLIs.\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/08_style.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/transform.py\n", + "def __getattr__(name):\n", + " raise ImportError(\n", + " f\"Could not import '{name}' from fastcore.transform - this module has been moved to the fasttransform package.\\n\"\n", + " \"To migrate your code, please see the migration guide at: https://answerdotai.github.io/fasttransform/fastcore_migration_guide.html\"\n", + " )\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/utils.py\n", + "from .imports import *\n", + "from .foundation import *\n", + "from .basics import *\n", + "from .xtras import *\n", + "from .parallel import *\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/xdg.py\n", + "\"\"\"XDG Base Directory Specification helpers.\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/07_xdg.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/xml.py\n", + "\"\"\"Concise generation of XML.\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/09_xml.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/xtras.py\n", + "\"\"\"Utility functions used in the fastai library\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/03_xtras.ipynb.\n", + "\n", + "# %% ../nbs/03_xtras.ipynb 1\n" + ] + } + ], + "source": [ + "for o in contents:\n", + " _head = \"\\n\".join(o.content_decoded.split(\"\\n\")[:5])\n", + " print(f\"{o.html_url}\\n{_head}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "contents = api.get_repo_contents(owner,\"ghapi\",branch=\"main\",inc=inc, exc=exc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "https://github.com/AnswerDotAI/ghapi/blob/main/.github/scripts/build-tweet.py\n", + "import tweetrel\n", + "tweetrel.send_tweet()\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/CHANGELOG.md\n", + "# Release notes\n", + "\n", + "\n", + "\n", + "## 1.0.6\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/CONTRIBUTING.md\n", + "# How to contribute\n", + "\n", + "## How to get started\n", + "\n", + "Before anything else, please install the git hooks that run automatic scripts during each commit and merge to strip the notebooks of superfluous metadata (and avoid merge conflicts). After cloning the repository, run the following command inside it:\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/README.md\n", + "# ghapi\n", + "\n", + "\n", + "\n", + "\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/examples/build.py\n", + "#!/usr/bin/env python\n", + "from ghapi.build_lib import *\n", + "build_funcs()\n", + "\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/actions.py\n", + "\"\"\"Functionality for helping to create GitHub Actions workflows in Python\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../01_actions.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/all.py\n", + "from .core import *\n", + "from .actions import *\n", + "from .auth import *\n", + "from .page import *\n", + "from .event import *\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/auth.py\n", + "\"\"\"Helpers for creating GitHub API tokens\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../02_auth.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/build_lib.py\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../90_build_lib.ipynb.\n", + "\n", + "# %% auto 0\n", + "__all__ = ['GH_OPENAPI_URL', 'GhMeta', 'build_funcs']\n", + "\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/cli.py\n", + "\"\"\"Access to the GitHub API from the command line\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../10_cli.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/core.py\n", + "\"\"\"Detailed information on the GhApi API\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../00_core.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/event.py\n", + "\"\"\"Helpers for getting GitHub API events\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../04_event.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/metadata.py\n", + "funcs = [('/', 'get', 'meta/root', 'GitHub API Root', 'rest/meta/meta#github-api-root', [], [], ''),\n", + " ('/advisories',\n", + " 'get',\n", + " 'security-advisories/list-global-advisories',\n", + " 'List global security advisories',\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/page.py\n", + "\"\"\"Parallel and serial pagination\"\"\"\n", + "\n", + "# AUTOGENERATED! DO NOT EDIT! File to edit: ../03_page.ipynb.\n", + "\n", + "# %% auto 0\n", + "https://github.com/AnswerDotAI/ghapi/blob/main/ghapi/templates.py\n", + "wf_tmpl = \"\"\"name: $NAME\n", + "on:\n", + " workflow_dispatch:\n", + "$EVENT\n", + "defaults:\n" + ] + } + ], + "source": [ + "for o in contents:\n", + " _head = \"\\n\".join(o.content_decoded.split(\"\\n\")[:5])\n", + " print(f\"{o.html_url}\\n{_head}\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1772,9 +2653,9 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "python", "language": "python", - "name": "python3" + "name": "python" } }, "nbformat": 4, diff --git a/ghapi/_modidx.py b/ghapi/_modidx.py index 549d942..03c52ae 100644 --- a/ghapi/_modidx.py +++ b/ghapi/_modidx.py @@ -64,6 +64,9 @@ 'ghapi.core.GhApi.full_docs': ('core.html#ghapi.full_docs', 'ghapi/core.py'), 'ghapi.core.GhApi.get_branch': ('core.html#ghapi.get_branch', 'ghapi/core.py'), 'ghapi.core.GhApi.get_content': ('core.html#ghapi.get_content', 'ghapi/core.py'), + 'ghapi.core.GhApi.get_file_content': ('core.html#ghapi.get_file_content', 'ghapi/core.py'), + 'ghapi.core.GhApi.get_repo_contents': ('core.html#ghapi.get_repo_contents', 'ghapi/core.py'), + 'ghapi.core.GhApi.get_repo_files': ('core.html#ghapi.get_repo_files', 'ghapi/core.py'), 'ghapi.core.GhApi.list_branches': ('core.html#ghapi.list_branches', 'ghapi/core.py'), 'ghapi.core.GhApi.list_files': ('core.html#ghapi.list_files', 'ghapi/core.py'), 'ghapi.core.GhApi.list_tags': ('core.html#ghapi.list_tags', 'ghapi/core.py'), @@ -82,6 +85,8 @@ 'ghapi.core._GhVerbGroup.__str__': ('core.html#_ghverbgroup.__str__', 'ghapi/core.py'), 'ghapi.core._GhVerbGroup._repr_markdown_': ('core.html#_ghverbgroup._repr_markdown_', 'ghapi/core.py'), 'ghapi.core._decode_response': ('core.html#_decode_response', 'ghapi/core.py'), + 'ghapi.core._find_matches': ('core.html#_find_matches', 'ghapi/core.py'), + 'ghapi.core._include': ('core.html#_include', 'ghapi/core.py'), 'ghapi.core._mk_param': ('core.html#_mk_param', 'ghapi/core.py'), 'ghapi.core._mk_sig': ('core.html#_mk_sig', 'ghapi/core.py'), 'ghapi.core._mk_sig_detls': ('core.html#_mk_sig_detls', 'ghapi/core.py'), diff --git a/ghapi/core.py b/ghapi/core.py index f251fb8..5343a5b 100644 --- a/ghapi/core.py +++ b/ghapi/core.py @@ -17,7 +17,7 @@ from datetime import datetime,timedelta from pprint import pprint from time import sleep -import os +import os,fnmatch # %% ../00_core.ipynb 4 GH_HOST = os.getenv('GH_HOST', "https://api.github.com") @@ -281,7 +281,54 @@ def update_contents(self:GhApi, path, message, committer, author, content, sha=N if sha is None: sha = self.list_files()[path].sha return self.create_or_update_file(path, message, committer=committer, author=author, content=content, sha=sha, branch=branch) -# %% ../00_core.ipynb 117 +# %% ../00_core.ipynb 121 +def _find_matches(path, pats): + "Returns matched patterns" + matches = [] + for p in listify(pats): + if fnmatch.fnmatch(path,p): matches.append(p) + return matches + +# %% ../00_core.ipynb 123 +def _include(path, include, exclude): + "Prioritize non-star matches, if both include and exclude star expr then pick longer." + include_matches = ["*"] if include is None else _find_matches(path, include) + exclude_matches = [] if exclude is None else _find_matches(path, exclude) + if include_matches and exclude_matches: + include_star = [m for m in include_matches if "*" in m] + exclude_star = [m for m in exclude_matches if "*" in m] + if include_star and exclude_star: return len(include_star) > len(exclude_star) + if include_star: return False + if exclude_star: return True + if include_matches: return True + if exclude_matches: return False + +# %% ../00_core.ipynb 140 +@patch +def get_repo_files(self:GhApi, owner, repo, branch="main", inc=None, exc=None): + "Get all file items of a repo." + tree = self.git.get_tree(owner=owner, repo=repo, tree_sha=branch, recursive=True) + res = L() + for item in tree['tree']: + if item['type'] == 'blob': res.append(item) + return res.filter(lambda o: _include(o.path,inc,exc)) + +# %% ../00_core.ipynb 142 +@patch +def get_file_content(self:GhApi, path, owner, repo, branch="main"): + o = self.repos.get_content(owner, repo, path, ref=branch) + o['content_decoded'] = base64.b64decode(o.content).decode('utf-8') + return o + +# %% ../00_core.ipynb 145 +@patch +@delegates(GhApi.get_repo_files) +def get_repo_contents(self:GhApi, owner, repo, **kwargs): + repo_files = self.get_repo_files(owner, repo, **kwargs) + for s in ('inc','exc',): kwargs.pop(s) + return parallel(self.get_file_content, repo_files.attrgot("path"), owner=owner, repo=repo, **kwargs) + +# %% ../00_core.ipynb 152 @patch def enable_pages(self:GhApi, branch=None, path="/"): "Enable or update pages for a repo to point to a `branch` and `path`."