Support GitHub Enterprise

GitHub can be hosted by companies for their own usage (under their own
URL).
This changset removes the hard-coded url to `github.com` and makes it
confligurable for a user, which GitHub-instances shall be supported.

Closes #134

Author: castler

docs/reference.md

@@ -3356,6 +3356,7 @@ Parameter | Description
 Name | Type | Description
 ---- | ---- | -----------
 <span style="white-space: nowrap;">`--allstar-app-ids`</span> | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--gql-commit-history-override`</span> | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR
@@ -3404,6 +3405,7 @@ Name | Type | Description
 <span style="white-space: nowrap;">`--git-destination-push`</span> | *string* | If set, overrides the git destination push reference.
 <span style="white-space: nowrap;">`--git-destination-url`</span> | *string* | If set, overrides the git destination URL.
 <span style="white-space: nowrap;">`--git-skip-checker`</span> | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--gql-commit-history-override`</span> | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR
@@ -3444,6 +3446,7 @@ Name | Type | Description
 <span style="white-space: nowrap;">`--git-fuzzy-last-rev`</span> | *boolean* | By default Copybara will try to migrate the revision listed as the version in the metadata file from github. This flag tells Copybara to first find the git tag which most closely matches the metadata version, and use that for the migration.
 <span style="white-space: nowrap;">`--git-origin-log-batch`</span> | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB.
 <span style="white-space: nowrap;">`--git-origin-rebase-ref`</span> | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--gql-commit-history-override`</span> | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR
@@ -3539,6 +3542,7 @@ Name | Type | Description
 <span style="white-space: nowrap;">`--git-destination-push`</span> | *string* | If set, overrides the git destination push reference.
 <span style="white-space: nowrap;">`--git-destination-url`</span> | *string* | If set, overrides the git destination URL.
 <span style="white-space: nowrap;">`--git-skip-checker`</span> | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--github-destination-pr-branch`</span> | *string* | If set, uses this branch for creating the pull request instead of using a generated one
@@ -3605,6 +3609,7 @@ Name | Type | Description
 <span style="white-space: nowrap;">`--git-fuzzy-last-rev`</span> | *boolean* | By default Copybara will try to migrate the revision listed as the version in the metadata file from github. This flag tells Copybara to first find the git tag which most closely matches the metadata version, and use that for the migration.
 <span style="white-space: nowrap;">`--git-origin-log-batch`</span> | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB.
 <span style="white-space: nowrap;">`--git-origin-rebase-ref`</span> | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--github-force-import`</span> | *boolean* | Force import regardless of the state of the PR
@@ -3644,6 +3649,7 @@ Parameter | Description
 Name | Type | Description
 ---- | ---- | -----------
 <span style="white-space: nowrap;">`--allstar-app-ids`</span> | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar.
+<span style="white-space: nowrap;">`--github-allowed-hosts`</span> | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported.
 <span style="white-space: nowrap;">`--github-api-bearer-auth`</span> | *boolean* | If using a token for GitHub access, bearer auth might be required
 <span style="white-space: nowrap;">`--github-destination-delete-pr-branch`</span> | *boolean* | Overwrite git.github_destination delete_pr_branch field
 <span style="white-space: nowrap;">`--gql-commit-history-override`</span> | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR

java/com/google/copybara/git/BUILD

@@ -73,6 +73,7 @@ java_library(
         "//java/com/google/copybara/monitor",
         "//java/com/google/copybara/profiler",
         "//java/com/google/copybara/revision",
+        "//java/com/google/copybara/starlark",
         "//java/com/google/copybara/templatetoken",
         "//java/com/google/copybara/transform",
         "//java/com/google/copybara/transform/patch",

java/com/google/copybara/git/GitHubOptions.java

@@ -39,7 +39,10 @@
 import com.google.copybara.jcommander.DurationConverter;
 import com.google.copybara.jcommander.GreaterThanZeroListValidator;
 import com.google.copybara.jcommander.SemicolonSeparatedListSplitter;
+import com.google.copybara.starlark.StarlarkUtil;
 import com.google.copybara.util.console.Console;
+import net.starlark.java.eval.EvalException;
+
 import java.io.IOException;
 import java.time.Duration;
 import java.util.List;
@@ -86,6 +89,27 @@ public class GitHubOptions implements Option {
       arity = 1)
   public boolean gitHubApiBearerAuth = false;
 
+  @Parameter(
+      names = "--github-allowed-hosts",
+      description = "If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported."
+  )
+  public List<String> gitHubAllowedHosts = ImmutableList.of("github.com");
+
+  public GitHubHost getGitHubHost(String url) throws EvalException  {
+      GitHubHost host = GitHubHost.fromUrl(url);
+      StarlarkUtil.check(gitHubAllowedHosts.contains(host.getHost()), "'%s' is not a valid GitHub url", host.getHost());
+      return host;
+  }
+
+  public boolean isGithubUrl(String url)
+  {
+      GitHubHost host = GitHubHost.fromUrl(url);
+      if(gitHubAllowedHosts.contains(host.getHost())){
+          return host.isGitHubUrl(url);
+      }
+
+      return false;
+  }
 
   public GitHubOptions(GeneralOptions generalOptions, GitOptions gitOptions) {
     this.generalOptions = Preconditions.checkNotNull(generalOptions);
@@ -100,7 +124,7 @@ public LazyResourceLoader<GitHubApi> newGitHubApiSupplier(
       GitHubHost ghHost) {
     return (console) -> {
       String project = ghHost.getProjectNameFromUrl(url);
-      return newGitHubRestApi(project, checker, credentials, console);
+      return newGitHubRestApi(ghHost, project, checker, credentials, console);
     };
   }
 
@@ -112,7 +136,7 @@ public LazyResourceLoader<GitHubGraphQLApi> newGitHubGraphQLApiSupplier(
       GitHubHost ghHost) {
     return (console) -> {
       String project = ghHost.getProjectNameFromUrl(url);
-      return newGitHubGraphQLApi(project, checker, credentials, console);
+      return newGitHubGraphQLApi(ghHost, project, checker, credentials, console);
     };
   }
 
@@ -121,9 +145,9 @@ public LazyResourceLoader<GitHubGraphQLApi> newGitHubGraphQLApiSupplier(
    *
    * <p>The project for 'https://github.com/foo/bar' is 'foo/bar'.
    */
-  public GitHubApi newGitHubRestApi(
+  public GitHubApi newGitHubRestApi(GitHubHost ghHost,
       String gitHubProject, @Nullable CredentialFileHandler credentials) throws RepoException {
-    return newGitHubRestApi(
+    return newGitHubRestApi(ghHost,
         gitHubProject, /* checker= */ null, credentials, generalOptions.console());
   }
 
@@ -134,6 +158,7 @@ public GitHubApi newGitHubRestApi(
    * <p>The project for 'https://github.com/foo/bar' is 'foo/bar'.
    */
   public GitHubApi newGitHubRestApi(
+      GitHubHost ghHost,
       String gitHubProject,
       @Nullable Checker checker,
       @Nullable CredentialFileHandler credentials,
@@ -144,7 +169,7 @@ public GitHubApi newGitHubRestApi(
     if (storePath == null) {
       storePath = "~/.git-credentials";
     }
-    GitHubApiTransport transport = newTransport(repo, storePath, console);
+    GitHubApiTransport transport = newTransport(ghHost, repo, storePath, console);
     if (checker != null) {
       transport = new GitHubApiTransportWithChecker(transport, new ApiChecker(checker, console));
     }
@@ -156,9 +181,9 @@ public GitHubApi newGitHubRestApi(
    *
    * <p>The project for 'https://github.com/foo/bar' is 'foo/bar'.
    */
-  public GitHubGraphQLApi newGitHubGraphQLApi(
+  public GitHubGraphQLApi newGitHubGraphQLApi(GitHubHost ghHost,
       String gitHubProject, @Nullable CredentialFileHandler credentials) throws RepoException {
-    return newGitHubGraphQLApi(
+    return newGitHubGraphQLApi(ghHost,
         gitHubProject, /* checker= */ null, credentials, generalOptions.console());
   }
 
@@ -169,6 +194,7 @@ public GitHubGraphQLApi newGitHubGraphQLApi(
    * <p>The project for 'https://github.com/foo/bar' is 'foo/bar'.
    */
   public GitHubGraphQLApi newGitHubGraphQLApi(
+      GitHubHost ghHost,
       String gitHubProject,
       @Nullable Checker checker,
       @Nullable CredentialFileHandler credentials,
@@ -180,7 +206,7 @@ public GitHubGraphQLApi newGitHubGraphQLApi(
     if (storePath == null) {
       storePath = "~/.git-credentials";
     }
-    GitHubApiTransport transport = newTransport(repo, storePath, console);
+    GitHubApiTransport transport = newTransport(ghHost, repo, storePath, console);
     if (checker != null) {
       transport = new GitHubApiTransportWithChecker(transport, new ApiChecker(checker, console));
     }
@@ -210,9 +236,9 @@ public void validateEndpointChecker(@Nullable Checker checker) throws Validation
     // Accept any by default
   }
 
-  private GitHubApiTransport newTransport(
+  private GitHubApiTransport newTransport(GitHubHost ghHost,
       GitRepository repo, String storePath, Console console) {
-    return new GitHubApiTransportImpl(
+    return new GitHubApiTransportImpl(ghHost,
         repo, newHttpTransport(), storePath, gitHubApiBearerAuth, console);
   }
 

java/com/google/copybara/git/GitHubPrDestination.java

@@ -234,7 +234,7 @@ public ImmutableList<DestinationEffect> write(
           return result.build();
         }
 
-        GitHubApi api = gitHubOptions.newGitHubRestApi(getProjectName(), credentials);
+        GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, getProjectName(), credentials);
 
         ImmutableList<PullRequest> pullRequests =
             api.getPullRequests(
@@ -346,7 +346,7 @@ public Endpoint getFeedbackEndPoint(Console console) throws ValidationException
   }
 
   private String asHttpsUrl() throws ValidationException {
-    return "https://github.com/" + getProjectName();
+    return ghHost.projectAsUrl(getProjectName());
   }
 
   @VisibleForTesting

java/com/google/copybara/git/GitHubPrOrigin.java

@@ -269,7 +269,7 @@ public String showDiff(GitRevision revisionFrom, GitRevision revisionTo) throws
   /** Given a commit SHA, use the GitHub API to (try to) look up info for a corresponding PR. */
   private PullRequest getPrFromSha(String project, String sha)
       throws RepoException, ValidationException {
-    GitHubApi gitHubApi = gitHubOptions.newGitHubRestApi(project, credentials);
+    GitHubApi gitHubApi = gitHubOptions.newGitHubRestApi(ghHost, project, credentials);
     IssuesAndPullRequestsSearchResults searchResults =
         gitHubApi.getIssuesOrPullRequestsSearchResults(
             new IssuesAndPullRequestsSearchRequestParams(
@@ -299,13 +299,13 @@ private PullRequest getPrFromSha(String project, String sha)
   private PullRequest getPrFromNumber(String project, long prNumber)
       throws RepoException, ValidationException {
     try (ProfilerTask ignore = generalOptions.profiler().start("github_api_get_pr")) {
-      return gitHubOptions.newGitHubRestApi(project, credentials).getPullRequest(project, prNumber);
+      return gitHubOptions.newGitHubRestApi(ghHost, project, credentials).getPullRequest(project, prNumber);
     }
   }
 
   private GitRevision getRevisionForPR(String project, PullRequest prData)
       throws RepoException, ValidationException {
-    GitHubApi api = gitHubOptions.newGitHubRestApi(project, credentials);
+    GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, project, credentials);
     int prNumber = (int) prData.getNumber();
     boolean actuallyUseMerge = this.useMerge;
     ImmutableListMultimap.Builder<String, String> labels = ImmutableListMultimap.builder();

java/com/google/copybara/git/GitHubPrWriteHook.java

@@ -95,7 +95,7 @@ public void beforePush(
     }
     for (Change<?> originalChange : originChanges) {
       String projectName = ghHost.getProjectNameFromUrl(repoUrl);
-      GitHubApi api = gitHubOptions.newGitHubRestApi(projectName, creds);
+      GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, projectName, creds);
 
       try {
         ImmutableList<PullRequest> pullRequests =

java/com/google/copybara/git/GitHubPreSubmitApprovalsProvider.java

@@ -202,7 +202,7 @@ public ImmutableList<ChangeWithApprovals> tryPresubmitUserValidation(
         ImmutableList.builder();
     ImmutableList<Review> reviews = null;
     try {
-      reviews = this.githubOptions.newGitHubRestApi(projectId, creds)
+      reviews = this.githubOptions.newGitHubRestApi(githubHost, projectId, creds)
           .getReviews(projectId, prNumber);
     } catch (RepoException | ValidationException e) {
       console.warnFmt(

java/com/google/copybara/git/GitHubWriteHook.java

@@ -92,7 +92,7 @@ public class GitHubWriteHook extends DefaultWriteHook {
   private PullRequest getPrFromNumber(String project, long prNumber)
       throws RepoException, ValidationException {
     try (ProfilerTask ignore = generalOptions.profiler().start("github_api_get_pr")) {
-      return gitHubOptions.newGitHubRestApi(project, creds).getPullRequest(project, prNumber);
+      return gitHubOptions.newGitHubRestApi(ghHost, project, creds).getPullRequest(project, prNumber);
     }
   }
 
@@ -106,7 +106,7 @@ public void beforePush(
       throws ValidationException, RepoException {
 
     String configProjectName = ghHost.getProjectNameFromUrl(repoUrl);
-    GitHubApi api = gitHubOptions.newGitHubRestApi(configProjectName, creds);
+    GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, configProjectName, creds);
 
 
     // TODO(joshgoldman): add credentials to the GitRepository object for pushing to the fork
@@ -204,7 +204,7 @@ public ImmutableList<DestinationEffect> afterPush(String serverResponse, Message
       return baseEffects.build();
     }
     String projectId = ghHost.getProjectNameFromUrl(repoUrl);
-    GitHubApi api = gitHubOptions.newGitHubRestApi(projectId, creds);
+    GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, projectId, creds);
 
     if (!originChanges.isEmpty()) {
       if (gitHubOptions.githubPrBranchDeletionDelay != null) {