Add pristine-lfs support

docs/common.ent

@@ -71,6 +71,7 @@
   <!ENTITY pbuilder    "<productname>Pbuilder</productname>">
   <!ENTITY pk4         "<productname>pk4</productname>">
   <!ENTITY pristine-tar "<productname>pristine-tar</productname>">
+  <!ENTITY pristine-lfs "<productname>pristine-lfs</productname>">
   <!ENTITY svn-buildpackage "<productname>svn-buildpackage</productname>">
 
   <!ENTITY changelog " <filename>debian/changelog</filename>">

docs/manpages/gbp-buildpackage.xml

@@ -60,6 +60,8 @@
       <arg><option>--git-export=</option><replaceable>TREEISH</replaceable></arg>
       <arg><option>--git-[no-]pristine-tar</option></arg>
       <arg><option>--git-[no-]pristine-tar-commit</option></arg>
+      <arg><option>--git-[no-]pristine-lfs</option></arg>
+      <arg><option>--git-[no-]pristine-lfs-commit</option></arg>
       <arg><option>--git-[no-]-purge</option></arg>
       <arg><option>--git-tag-only</option></arg>
       <arg><option>--git-retag</option></arg>
@@ -105,7 +107,7 @@
       </listitem>
       <listitem>
         <para>
-          Build an orig tarball if it doesn't exist. Optionally using &pristine-tar;.
+          Build an orig tarball if it doesn't exist. Optionally using &pristine-lfs; or &pristine-tar;.
         </para>
       </listitem>
       <listitem>
@@ -150,7 +152,8 @@
     <refsect2>
       <title>Upstream tarball creation options</title>
       <para>When &gbp-buildpackage; doesn't find a suitable upstream
-	tarball it will create one either using &pristine-tar;
+        tarball, it will first try to check it out from &pristine-lfs;. If &pristine-lfs;
+        doesn't produce a tarball, &gbp-buildpackage; will create one either using &pristine-tar;
 	or <command>git archive</command>. These options determine how the tarball is created:
       </para>
       <variablelist>
@@ -164,6 +167,26 @@
               the <option>--git-upstream-tag</option>, <option>--git-upstream-tree</option>
               options have no effect when creating tarballs.
             </para>
+            <para>
+              If both <option>--git-pristine-tar</option> and <option>--git-pristine-lfs</option>
+              are specified, <option>--git-pristine-lfs</option> takes precedence.
+            </para>
+          </listitem>
+        </varlistentry>
+	<varlistentry>
+          <term><option>--git-pristine-lfs</option>
+          </term>
+          <listitem>
+            <para>
+              Use pristine-lfs when generating the upstream tarball if
+              it doesn't exist. If this mode is enabled
+              the <option>--git-upstream-tag</option>, <option>--git-upstream-tree</option>
+              options have no effect when creating tarballs.
+            </para>
+            <para>
+              If both <option>--git-pristine-tar</option> and <option>--git-pristine-lfs</option>
+              are specified, <option>--git-pristine-lfs</option> takes precedence.
+            </para>
           </listitem>
 	</varlistentry>
 	<varlistentry>
@@ -256,6 +279,16 @@
               tarball was generated and the pristine-tar data isn't already there.
             </para>
           </listitem>
+        </varlistentry>
+	<varlistentry>
+          <term><option>--git-pristine-lfs-commit</option>
+          </term>
+          <listitem>
+            <para>
+              Commit the tarball to the pristine-lfs branch if a new
+              tarball was generated and is hasn't been already imported there.
+            </para>
+          </listitem>
 	</varlistentry>
 	<varlistentry>
           <term><option>--git-force-create</option>

docs/manpages/gbp-export-orig.xml

@@ -32,6 +32,8 @@
       <arg rep='repeat'><option>--component=</option><replaceable>component</replaceable></arg>
       <arg><option>--[no-]pristine-tar</option></arg>
       <arg><option>--[no-]pristine-tar-commit</option></arg>
+      <arg><option>--[no-]pristine-lfs</option></arg>
+      <arg><option>--[no-]pristine-lfs-commit</option></arg>
       <arg><option>--upstream-signatures=</option>[auto|on|off]</arg>
     </cmdsynopsis>
   </refsynopsisdiv>
@@ -137,7 +139,7 @@
           </para>
           <para>
           This doesn't have any effect if <option>--pristine-tar</option>
-          is being used.
+          or <option>--pristine-lfs</option> is being used.
           </para>
         </listitem>
       </varlistentry>
@@ -203,7 +205,8 @@
         <listitem>
           <para>
           Use pristine-tar when generating the upstream tarball if it doesn't
-          exist.
+            exist. If used with <option>--pristine-lfs</option>, pristine-tar
+            is only used when pristine-lfs didn’t produce a tarball.
           </para>
         </listitem>
       </varlistentry>
@@ -217,6 +220,27 @@
           </para>
         </listitem>
       </varlistentry>
+      <varlistentry>
+        <term><option>--pristine-lfs</option>
+        </term>
+        <listitem>
+          <para>
+          Use pristine-lfs when generating the upstream tarball if it doesn't
+            exist. If used with <option>--pristine-tar</option>, pristine-tar
+            is only used when pristine-lfs didn’t produce a tarball.
+          </para>
+        </listitem>
+      </varlistentry>
+      <varlistentry>
+        <term><option>--pristine-lfs-commit</option>
+        </term>
+        <listitem>
+          <para>
+            Commit the tarball to the pristine-lfs branch if a new
+            tarball was generated and is hasn't been already imported there.
+          </para>
+        </listitem>
+      </varlistentry>
     <varlistentry>
         <term><option>--upstream-signatures=</option>[auto|on|off]
         </term>

docs/manpages/gbp-import-dsc.xml

@@ -29,6 +29,7 @@
       <arg><option>--keyid=</option><replaceable>gpg-keyid</replaceable></arg>
       <arg><option>--[no-]create-missing-branches</option></arg>
       <arg><option>--[no-]pristine-tar</option></arg>
+      <arg><option>--[no-]pristine-lfs</option></arg>
       <arg><option>--[no-]sign-tags</option></arg>
       <arg><option>--skip-debian-tag</option></arg>
       <arg><option>--upstream-branch=</option><replaceable>branch_name</replaceable></arg>
@@ -192,6 +193,15 @@
           </para>
         </listitem>
       </varlistentry>
+      <varlistentry>
+        <term><option>--pristine-lfs</option>
+        </term>
+        <listitem>
+          <para>
+          Import tarballs into pristine-lfs.
+          </para>
+        </listitem>
+      </varlistentry>
       <varlistentry>
         <term><option>--allow-unauthenticated</option>
         </term>

docs/manpages/gbp-import-orig.xml

@@ -34,6 +34,7 @@
       <arg rep='repeat'><option>--component=</option><replaceable>component</replaceable></arg>
       <arg><option>--[no-]pristine-tar</option></arg>
       <arg><option>--[no-]filter-pristine-tar</option></arg>
+      <arg><option>--[no-]pristine-lfs</option></arg>
       <arg><option>--[no-]symlink-orig</option></arg>
       <arg><option>--postimport=cmd</option></arg>
       <arg><option>--postunpack=cmd</option></arg>
@@ -289,6 +290,15 @@
           </para>
         </listitem>
       </varlistentry>
+      <varlistentry>
+        <term><option>--[no-]pristine-lfs</option>
+        </term>
+        <listitem>
+          <para>
+          Import tarballs into <command>pristine-lfs</command>.
+          </para>
+        </listitem>
+      </varlistentry>
       <varlistentry>
         <term><option>--[no-]symlink-orig</option></term>
         <listitem>

docs/manpages/man.seealso.xml

@@ -10,5 +10,9 @@
         <refentrytitle>pristine-tar</refentrytitle>
         <manvolnum>1</manvolnum>
       </citerefentry>,
+      <citerefentry>
+        <refentrytitle>pristine-lfs</refentrytitle>
+        <manvolnum>1</manvolnum>
+      </citerefentry>,
       <ulink url="file:///usr/share/doc/git-buildpackage/manual-html/index.html">
       <citetitle>The Git-Buildpackage Manual</citetitle></ulink>

gbp/config.py

@@ -174,6 +174,8 @@ class GbpOptionParser(OptionParser):
                 'pq-from': 'DEBIAN',
                 'prebuild': '',
                 'preexport': '',
+                'pristine-lfs': 'False',
+                'pristine-lfs-commit': 'False',
                 'pristine-tar': 'False',
                 'pristine-tar-commit': 'False',
                 'purge': 'True',
@@ -238,6 +240,12 @@ class GbpOptionParser(OptionParser):
         'commit-msg':
             "Format string for commit message used to commit, "
             "the changelog, default is '%(commit-msg)s'",
+        'pristine-lfs':
+            "Use pristine-lfs to create orig tarball, "
+            "default is '%(pristine-lfs)s'",
+        'pristine-lfs-commit':
+            "When generating a tarball, commit it to the pristine-lfs branch '%(pristine-lfs-commit)s' "
+            "default is '%(pristine-lfs-commit)s'",
         'pristine-tar':
             "Use pristine-tar to create orig tarball, "
             "default is '%(pristine-tar)s'",

gbp/deb/git.py

@@ -22,6 +22,7 @@
 from gbp.command_wrappers import CommandExecFailed
 from gbp.git import GitRepositoryError
 from gbp.deb.pristinetar import DebianPristineTar
+from gbp.deb.pristinelfs import PristineLfs
 from gbp.paths import to_bin
 from gbp.pkg.git import PkgGitRepository
 from gbp.pkg.pkgpolicy import PkgPolicy
@@ -339,6 +340,43 @@ def create_upstream_tarball_via_pristine_tar(self, source, output_dir, comp, ups
                                                                   e))
         return True
 
+    @property
+    def pristine_lfs_branch(self):
+        """
+        The name of the pristine-lfs branch, whether it already exists or
+        not.
+        """
+        return PristineLfs.branch
+
+    def has_pristine_lfs_branch(self):
+        """
+        Whether the repo has a I{pristine-lfs} branch.
+
+        @return: C{True} if the repo has pristine-lfs commits already, C{False}
+            otherwise
+        @rtype: C{Bool}
+        """
+        return True if self.has_branch(self.pristine_lfs_branch) else False
+
+    def create_pristine_lfs_commits(self, sources):
+        """
+        Create pristine-lfs commits for a package with main tarball
+        and (optional) component tarballs based on upstream_tree
+
+        @param soures: C{list} of tarball as I{UpstreamSource}. First one being the main
+                       tarball the other ones additional tarballs.
+        """
+        all_files = [
+            source.path for source in sources
+        ] + [
+            source.signaturefile for source in sources if source.signaturefile
+        ]
+
+        try:
+            self.pristine_lfs.commit(all_files)
+        except CommandExecFailed as e:
+            raise GitRepositoryError(str(e))
+
     def create_upstream_tarball_via_git_archive(self, source, output_dir, treeish,
                                                 comp, with_submodules, component=None):
         """

gbp/deb/pristinelfs.py

@@ -0,0 +1,86 @@
+# vim: set fileencoding=utf-8 :
+#
+# (C) 2021 Andrej Shadura <[email protected]>
+# (C) 2021 Collabora Ltd
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program; if not, please see
+#    <http://www.gnu.org/licenses/>
+"""Check in and check out archives from the pristine-lfs branch"""
+
+from __future__ import annotations
+
+import logging
+
+from gbp.command_wrappers import CommandExecFailed
+from gbp.git import GitRepository, GitRepositoryError
+
+try:
+    from pristine_lfs.main import do_list, do_verify, do_commit_files, do_checkout
+    from pristine_lfs.errors import CommandFailed, DifferentFilesExist, GitError
+except ImportError:
+    def pristine_lfs_not_found(*args, **kwargs):
+        raise CommandExecFailed("pristine-lfs not installed")
+
+    do_list = do_verify = do_checkout = do_commit_files = pristine_lfs_not_found
+
+    class DifferentFilesExist(Exception):
+        pass
+
+    GitError = DifferentFilesExist
+
+logger = logging.getLogger('pristine-lfs')
+
+
+class PristineLfs:
+    branch = 'pristine-lfs'
+
+    def __init__(self, repo: GitRepository):
+        self.repo = repo
+
+    def commit(self, files: list[str], quiet: bool = False):
+        """
+        Commit files I{files} to the pristine-lfs branch
+
+        @param files: list of files to commit
+        @type files: C{list}
+        """
+        logger.setLevel(logging.WARNING if quiet else logging.INFO)
+
+        try:
+            ios = [open(f, 'rb') for f in files]
+            do_commit_files(tarballs=ios, branch=self.branch)
+        except (OSError, CommandFailed) as e:
+            raise CommandExecFailed(str(e))
+        except (DifferentFilesExist, GitError) as e:
+            raise GitRepositoryError(str(e))
+
+    def checkout(self, package: str, version: str, output_dir: str, quiet: bool = False):
+        """
+        Check out all orig tarballs for package I{package} of I{version} to
+        I{output_dir}
+
+        @param package: the package to check out the orig tarballs for
+        @type package: C{str}
+        @param version: the version to check out the orig tarballs for
+        @type version: C{str}
+        @param output_dir: the directory to put the tarballs into
+        @type output_dir: C{str}
+        """
+        logger.setLevel(logging.WARNING if quiet else logging.INFO)
+
+        try:
+            do_checkout(package=package, version=version, branch=self.branch, outdir=output_dir)
+        except (OSError, CommandFailed) as e:
+            raise CommandExecFailed(str(e))
+        except GitError as e:
+            raise GitRepositoryError(str(e))

gbp/pkg/git.py

@@ -24,6 +24,7 @@
 from gbp.command_wrappers import (CatenateTarArchive, CatenateZipArchive)
 from gbp.git import GitRepository, GitRepositoryError
 from gbp.deb.pristinetar import DebianPristineTar
+from gbp.deb.pristinelfs import PristineLfs
 
 
 import gbp.log
@@ -35,6 +36,7 @@ class PkgGitRepository(GitRepository):
     def __init__(self, *args, **kwargs):
         super(PkgGitRepository, self).__init__(*args, **kwargs)
         self.pristine_tar = DebianPristineTar(self)
+        self.pristine_lfs = PristineLfs(self)
 
     @staticmethod
     def sanitize_prefix(prefix):

gbp/scripts/buildpackage.py

@@ -385,6 +385,7 @@ def build_parser(name, prefix=None):
     tag_group.add_config_file_option(option_name="debian-tag-msg", dest="debian_tag_msg")
     tag_group.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
     orig_group.add_config_file_option(option_name="upstream-tree", dest="upstream_tree")
+    orig_group.add_boolean_config_file_option(option_name="pristine-lfs", dest="pristine_lfs")
     orig_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
     orig_group.add_boolean_config_file_option(option_name="pristine-tar-commit",
                                               dest="pristine_tar_commit")

gbp/scripts/clone.py

@@ -141,6 +141,7 @@ def build_parser(name):
     branch_group.add_config_file_option(option_name="upstream-branch", dest="upstream_branch")
     branch_group.add_config_file_option(option_name="debian-branch", dest="debian_branch")
     branch_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
+    branch_group.add_boolean_config_file_option(option_name="pristine-lfs", dest="pristine_lfs")
     branch_group.add_option("--depth", action="store", dest="depth", default=0,
                             help="git history depth (for creating shallow clones)")
     branch_group.add_option("--reference", action="store", dest="reference", default=None,
@@ -224,6 +225,8 @@ def main(argv):
             branches = [options.debian_branch, options.upstream_branch]
             if options.pristine_tar:
                 branches += [repo.pristine_tar_branch]
+            if options.pristine_lfs:
+                branches += [repo.pristine_lfs_branch]
             gbp.log.debug('Will track branches: %s' % branches)
             for branch in branches:
                 remote = 'origin/%s' % branch