From c4f4a502ced80fd1b63ca88545b4741884365edc Mon Sep 17 00:00:00 2001 From: Jan Schlosser Date: Wed, 5 Mar 2025 13:15:30 +0100 Subject: [PATCH 1/2] Support GitHub Enterprise GitHub can be hosted by companies for their own usage (under their own URL). This changset removes the hard-coded url to `github.com` and makes it confligurable for a user, which GitHub-instances shall be supported. Closes #134 --- docs/reference.md | 6 ++ java/com/google/copybara/git/BUILD | 1 + .../google/copybara/git/GitHubOptions.java | 46 ++++++-- .../copybara/git/GitHubPrDestination.java | 4 +- .../google/copybara/git/GitHubPrOrigin.java | 6 +- .../copybara/git/GitHubPrWriteHook.java | 2 +- .../git/GitHubPreSubmitApprovalsProvider.java | 2 +- .../google/copybara/git/GitHubWriteHook.java | 6 +- java/com/google/copybara/git/GitModule.java | 102 +++++++++++------- java/com/google/copybara/git/GitRepoType.java | 6 +- java/com/google/copybara/git/Mirror.java | 4 +- java/com/google/copybara/git/github/BUILD | 1 + .../github/api/GitHubApiTransportImpl.java | 31 +++--- .../copybara/git/github/util/GitHubHost.java | 21 ++++ java/com/google/copybara/rust/RustModule.java | 4 +- .../google/copybara/git/GitOriginTest.java | 22 +++- .../git/github/api/GitHubApiTest.java | 3 +- .../api/GitHubApiTransportImplTest.java | 9 +- .../git/github/api/GitHubGraphQLTest.java | 3 +- 19 files changed, 192 insertions(+), 87 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 03a97b466..49a4f50ed 100755 --- a/docs/reference.md +++ b/docs/reference.md @@ -3394,6 +3394,7 @@ Parameter | Description Name | Type | Description ---- | ---- | ----------- `--allstar-app-ids` | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3442,6 +3443,7 @@ Name | Type | Description `--git-destination-push` | *string* | If set, overrides the git destination push reference. `--git-destination-url` | *string* | If set, overrides the git destination URL. `--git-skip-checker` | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3483,6 +3485,7 @@ Name | Type | Description `--git-origin-log-batch` | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB. `--git-origin-non-linear-history` | *boolean* | Read the full git log and skip changes before the from ref rather than using a log path. `--git-origin-rebase-ref` | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3578,6 +3581,7 @@ Name | Type | Description `--git-destination-push` | *string* | If set, overrides the git destination push reference. `--git-destination-url` | *string* | If set, overrides the git destination URL. `--git-skip-checker` | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--github-destination-pr-branch` | *string* | If set, uses this branch for creating the pull request instead of using a generated one @@ -3645,6 +3649,7 @@ Name | Type | Description `--git-origin-log-batch` | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB. `--git-origin-non-linear-history` | *boolean* | Read the full git log and skip changes before the from ref rather than using a log path. `--git-origin-rebase-ref` | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--github-force-import` | *boolean* | Force import regardless of the state of the PR @@ -3684,6 +3689,7 @@ Parameter | Description Name | Type | Description ---- | ---- | ----------- `--allstar-app-ids` | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar. +`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR diff --git a/java/com/google/copybara/git/BUILD b/java/com/google/copybara/git/BUILD index 6a6c4d3e0..426165015 100644 --- a/java/com/google/copybara/git/BUILD +++ b/java/com/google/copybara/git/BUILD @@ -78,6 +78,7 @@ java_library( "//java/com/google/copybara/monitor", "//java/com/google/copybara/profiler", "//java/com/google/copybara/revision", + "//java/com/google/copybara/starlark", "//java/com/google/copybara/templatetoken", "//java/com/google/copybara/transform", "//java/com/google/copybara/transform/patch", diff --git a/java/com/google/copybara/git/GitHubOptions.java b/java/com/google/copybara/git/GitHubOptions.java index 20f65c3a0..cc567e62e 100644 --- a/java/com/google/copybara/git/GitHubOptions.java +++ b/java/com/google/copybara/git/GitHubOptions.java @@ -39,7 +39,10 @@ import com.google.copybara.jcommander.DurationConverter; import com.google.copybara.jcommander.GreaterThanZeroListValidator; import com.google.copybara.jcommander.SemicolonSeparatedListSplitter; +import com.google.copybara.starlark.StarlarkUtil; import com.google.copybara.util.console.Console; +import net.starlark.java.eval.EvalException; + import java.io.IOException; import java.time.Duration; import java.util.List; @@ -86,6 +89,27 @@ public class GitHubOptions implements Option { arity = 1) public boolean gitHubApiBearerAuth = false; + @Parameter( + names = "--github-allowed-hosts", + description = "If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported." + ) + public List gitHubAllowedHosts = ImmutableList.of("github.com"); + + public GitHubHost getGitHubHost(String url) throws EvalException { + GitHubHost host = GitHubHost.fromUrl(url); + StarlarkUtil.check(gitHubAllowedHosts.contains(host.getHost()), "'%s' is not a valid GitHub url", host.getHost()); + return host; + } + + public boolean isGithubUrl(String url) + { + GitHubHost host = GitHubHost.fromUrl(url); + if(gitHubAllowedHosts.contains(host.getHost())){ + return host.isGitHubUrl(url); + } + + return false; + } public GitHubOptions(GeneralOptions generalOptions, GitOptions gitOptions) { this.generalOptions = Preconditions.checkNotNull(generalOptions); @@ -100,7 +124,7 @@ public LazyResourceLoader newGitHubApiSupplier( GitHubHost ghHost) { return (console) -> { String project = ghHost.getProjectNameFromUrl(url); - return newGitHubRestApi(project, checker, credentials, console); + return newGitHubRestApi(ghHost, project, checker, credentials, console); }; } @@ -112,7 +136,7 @@ public LazyResourceLoader newGitHubGraphQLApiSupplier( GitHubHost ghHost) { return (console) -> { String project = ghHost.getProjectNameFromUrl(url); - return newGitHubGraphQLApi(project, checker, credentials, console); + return newGitHubGraphQLApi(ghHost, project, checker, credentials, console); }; } @@ -121,9 +145,9 @@ public LazyResourceLoader newGitHubGraphQLApiSupplier( * *

The project for 'https://github.com/foo/bar' is 'foo/bar'. */ - public GitHubApi newGitHubRestApi( + public GitHubApi newGitHubRestApi(GitHubHost ghHost, String gitHubProject, @Nullable CredentialFileHandler credentials) throws RepoException { - return newGitHubRestApi( + return newGitHubRestApi(ghHost, gitHubProject, /* checker= */ null, credentials, generalOptions.console()); } @@ -134,6 +158,7 @@ public GitHubApi newGitHubRestApi( *

The project for 'https://github.com/foo/bar' is 'foo/bar'. */ public GitHubApi newGitHubRestApi( + GitHubHost ghHost, String gitHubProject, @Nullable Checker checker, @Nullable CredentialFileHandler credentials, @@ -144,7 +169,7 @@ public GitHubApi newGitHubRestApi( if (storePath == null) { storePath = "~/.git-credentials"; } - GitHubApiTransport transport = newTransport(repo, storePath, console); + GitHubApiTransport transport = newTransport(ghHost, repo, storePath, console); if (checker != null) { transport = new GitHubApiTransportWithChecker(transport, new ApiChecker(checker, console)); } @@ -156,9 +181,9 @@ public GitHubApi newGitHubRestApi( * *

The project for 'https://github.com/foo/bar' is 'foo/bar'. */ - public GitHubGraphQLApi newGitHubGraphQLApi( + public GitHubGraphQLApi newGitHubGraphQLApi(GitHubHost ghHost, String gitHubProject, @Nullable CredentialFileHandler credentials) throws RepoException { - return newGitHubGraphQLApi( + return newGitHubGraphQLApi(ghHost, gitHubProject, /* checker= */ null, credentials, generalOptions.console()); } @@ -169,6 +194,7 @@ public GitHubGraphQLApi newGitHubGraphQLApi( *

The project for 'https://github.com/foo/bar' is 'foo/bar'. */ public GitHubGraphQLApi newGitHubGraphQLApi( + GitHubHost ghHost, String gitHubProject, @Nullable Checker checker, @Nullable CredentialFileHandler credentials, @@ -180,7 +206,7 @@ public GitHubGraphQLApi newGitHubGraphQLApi( if (storePath == null) { storePath = "~/.git-credentials"; } - GitHubApiTransport transport = newTransport(repo, storePath, console); + GitHubApiTransport transport = newTransport(ghHost, repo, storePath, console); if (checker != null) { transport = new GitHubApiTransportWithChecker(transport, new ApiChecker(checker, console)); } @@ -210,9 +236,9 @@ public void validateEndpointChecker(@Nullable Checker checker) throws Validation // Accept any by default } - private GitHubApiTransport newTransport( + private GitHubApiTransport newTransport(GitHubHost ghHost, GitRepository repo, String storePath, Console console) { - return new GitHubApiTransportImpl( + return new GitHubApiTransportImpl(ghHost, repo, newHttpTransport(), storePath, gitHubApiBearerAuth, console); } diff --git a/java/com/google/copybara/git/GitHubPrDestination.java b/java/com/google/copybara/git/GitHubPrDestination.java index 5df1d4ab3..3278230ff 100644 --- a/java/com/google/copybara/git/GitHubPrDestination.java +++ b/java/com/google/copybara/git/GitHubPrDestination.java @@ -234,7 +234,7 @@ public ImmutableList write( return result.build(); } - GitHubApi api = gitHubOptions.newGitHubRestApi(getProjectName(), credentials); + GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, getProjectName(), credentials); ImmutableList pullRequests = api.getPullRequests( @@ -346,7 +346,7 @@ public Endpoint getFeedbackEndPoint(Console console) throws ValidationException } private String asHttpsUrl() throws ValidationException { - return "https://github.com/" + getProjectName(); + return ghHost.projectAsUrl(getProjectName()); } @VisibleForTesting diff --git a/java/com/google/copybara/git/GitHubPrOrigin.java b/java/com/google/copybara/git/GitHubPrOrigin.java index 061ddb7e7..17144c771 100644 --- a/java/com/google/copybara/git/GitHubPrOrigin.java +++ b/java/com/google/copybara/git/GitHubPrOrigin.java @@ -269,7 +269,7 @@ public String showDiff(GitRevision revisionFrom, GitRevision revisionTo) throws /** Given a commit SHA, use the GitHub API to (try to) look up info for a corresponding PR. */ private PullRequest getPrFromSha(String project, String sha) throws RepoException, ValidationException { - GitHubApi gitHubApi = gitHubOptions.newGitHubRestApi(project, credentials); + GitHubApi gitHubApi = gitHubOptions.newGitHubRestApi(ghHost, project, credentials); IssuesAndPullRequestsSearchResults searchResults = gitHubApi.getIssuesOrPullRequestsSearchResults( new IssuesAndPullRequestsSearchRequestParams( @@ -299,13 +299,13 @@ private PullRequest getPrFromSha(String project, String sha) private PullRequest getPrFromNumber(String project, long prNumber) throws RepoException, ValidationException { try (ProfilerTask ignore = generalOptions.profiler().start("github_api_get_pr")) { - return gitHubOptions.newGitHubRestApi(project, credentials).getPullRequest(project, prNumber); + return gitHubOptions.newGitHubRestApi(ghHost, project, credentials).getPullRequest(project, prNumber); } } private GitRevision getRevisionForPR(String project, PullRequest prData) throws RepoException, ValidationException { - GitHubApi api = gitHubOptions.newGitHubRestApi(project, credentials); + GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, project, credentials); int prNumber = (int) prData.getNumber(); boolean actuallyUseMerge = this.useMerge; ImmutableListMultimap.Builder labels = ImmutableListMultimap.builder(); diff --git a/java/com/google/copybara/git/GitHubPrWriteHook.java b/java/com/google/copybara/git/GitHubPrWriteHook.java index 4edc28f23..e763cabd9 100644 --- a/java/com/google/copybara/git/GitHubPrWriteHook.java +++ b/java/com/google/copybara/git/GitHubPrWriteHook.java @@ -95,7 +95,7 @@ public void beforePush( } for (Change originalChange : originChanges) { String projectName = ghHost.getProjectNameFromUrl(repoUrl); - GitHubApi api = gitHubOptions.newGitHubRestApi(projectName, creds); + GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, projectName, creds); try { ImmutableList pullRequests = diff --git a/java/com/google/copybara/git/GitHubPreSubmitApprovalsProvider.java b/java/com/google/copybara/git/GitHubPreSubmitApprovalsProvider.java index d20b963f2..148217f07 100644 --- a/java/com/google/copybara/git/GitHubPreSubmitApprovalsProvider.java +++ b/java/com/google/copybara/git/GitHubPreSubmitApprovalsProvider.java @@ -202,7 +202,7 @@ public ImmutableList tryPresubmitUserValidation( ImmutableList.builder(); ImmutableList reviews = null; try { - reviews = this.githubOptions.newGitHubRestApi(projectId, creds) + reviews = this.githubOptions.newGitHubRestApi(githubHost, projectId, creds) .getReviews(projectId, prNumber); } catch (RepoException | ValidationException e) { console.warnFmt( diff --git a/java/com/google/copybara/git/GitHubWriteHook.java b/java/com/google/copybara/git/GitHubWriteHook.java index 6b63c76c7..5c3d89505 100644 --- a/java/com/google/copybara/git/GitHubWriteHook.java +++ b/java/com/google/copybara/git/GitHubWriteHook.java @@ -92,7 +92,7 @@ public class GitHubWriteHook extends DefaultWriteHook { private PullRequest getPrFromNumber(String project, long prNumber) throws RepoException, ValidationException { try (ProfilerTask ignore = generalOptions.profiler().start("github_api_get_pr")) { - return gitHubOptions.newGitHubRestApi(project, creds).getPullRequest(project, prNumber); + return gitHubOptions.newGitHubRestApi(ghHost, project, creds).getPullRequest(project, prNumber); } } @@ -106,7 +106,7 @@ public void beforePush( throws ValidationException, RepoException { String configProjectName = ghHost.getProjectNameFromUrl(repoUrl); - GitHubApi api = gitHubOptions.newGitHubRestApi(configProjectName, creds); + GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, configProjectName, creds); // TODO(joshgoldman): add credentials to the GitRepository object for pushing to the fork @@ -204,7 +204,7 @@ public ImmutableList afterPush(String serverResponse, Message return baseEffects.build(); } String projectId = ghHost.getProjectNameFromUrl(repoUrl); - GitHubApi api = gitHubOptions.newGitHubRestApi(projectId, creds); + GitHubApi api = gitHubOptions.newGitHubRestApi(ghHost, projectId, creds); if (!originChanges.isEmpty()) { if (gitHubOptions.githubPrBranchDeletionDelay != null) { diff --git a/java/com/google/copybara/git/GitModule.java b/java/com/google/copybara/git/GitModule.java index 55f1cf304..385a015e0 100644 --- a/java/com/google/copybara/git/GitModule.java +++ b/java/com/google/copybara/git/GitModule.java @@ -85,6 +85,7 @@ import com.google.copybara.git.github.api.CheckRun.Conclusion; import com.google.copybara.git.github.api.GitHubEventType; import com.google.copybara.git.github.api.GitHubGraphQLApi.GetCommitHistoryParams; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.git.github.util.GitHubUtil; import com.google.copybara.git.gitlab.GitLabOptions; import com.google.copybara.git.gitlab.api.entities.MergeRequest.DetailedMergeStatus; @@ -353,6 +354,10 @@ public GitOrigin origin( checkSubmoduleConfig(submodules, excludedSubmoduleList); String fixedUrl = fixHttp(url, thread.getCallerLocation()); CredentialFileHandler credentialHandler = getCredentialHandler(fixedUrl, credentials); + + GitHubOptions githubOptions = options.get(GitHubOptions.class); + boolean isGitHubUrl = githubOptions.isGithubUrl(url); + return GitOrigin.newGitOrigin( options, fixedUrl, @@ -369,7 +374,7 @@ public GitOrigin origin( validateVersionSelector(versionSelector), mainConfigFile.path(), workflowName, - GITHUB_COM.isGitHubUrl(url) + isGitHubUrl ? githubPostSubmitApprovalsProvider( fixedUrl, SkylarkUtil.convertOptionalString(ref), credentialHandler) : approvalsProvider(url), @@ -1157,7 +1162,8 @@ public Origin githubPrOrigin( StarlarkThread thread) throws EvalException { checkNotEmpty(url, "url"); - check(GITHUB_COM.isGitHubUrl(url), "Invalid Github URL: %s", url); + GitHubOptions gitHubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = gitHubOptions.getGitHubHost(url); PatchTransformation patchTransformation = maybeGetPatchTransformation(patch); List excludedSubmoduleList = @@ -1248,7 +1254,7 @@ public Origin githubPrOrigin( patchTransformation, convertFromNoneable(branch, null), convertDescribeVersion(describeVersion), - GITHUB_COM, + ghHost, githubPreSubmitApprovalsProvider(fixedUrl, credHandler), credHandler); } @@ -1387,7 +1393,8 @@ public GitOrigin githubOrigin( @Nullable Object credentials, StarlarkThread thread) throws EvalException { - check(GITHUB_COM.isGitHubUrl(checkNotEmpty(url, "url")), "Invalid Github URL: %s", url); + GitHubOptions gitHubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = gitHubOptions.getGitHubHost(checkNotEmpty(url, "url")); if (versionSelector != Starlark.NONE) { check( @@ -1966,6 +1973,7 @@ public GitDestination gitHubDestination( branchToUpdate != null || deletePrBranch == null, "'delete_pr_branch' can only be set if 'pr_branch_to_update' is used"); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = gitHubOptions.getGitHubHost(url); WorkflowOptions workflowOptions = options.get(WorkflowOptions.class); String effectivePrBranchToUpdate = branchToUpdate; @@ -1987,7 +1995,7 @@ public GitDestination gitHubDestination( CredentialFileHandler credentialHandler; try { credentialHandler = getCredentialHandler( - GITHUB_COM.getHost(), GITHUB_COM.getProjectNameFromUrl(url), credentials); + ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { throw new EvalException("Cannot parse url", e); } @@ -2012,7 +2020,7 @@ public GitDestination gitHubDestination( effectiveDeletePrBranch, getGeneralConsole(), apiCheckerObj != null ? apiCheckerObj : checkerObj, - GITHUB_COM, + ghHost, credentialHandler, pushToFork), Starlark.isNullOrNone(integrates) @@ -2265,18 +2273,18 @@ public GitHubPrDestination githubPrDestination( StarlarkThread thread) throws EvalException { GeneralOptions generalOptions = options.get(GeneralOptions.class); - // This restricts to github.com, we will have to revisit this to support setups like GitHub - // Enterprise. - check(GITHUB_COM.isGitHubUrl(url), "'%s' is not a valid GitHub url", url); + GitDestinationOptions destinationOptions = options.get(GitDestinationOptions.class); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); String destinationPrBranch = convertFromNoneable(prBranch, null); Checker apiCheckerObj = convertFromNoneable(apiChecker, null); Checker checkerObj = convertFromNoneable(checker, null); CredentialFileHandler credentialHandler; + GitHubHost ghHost = gitHubOptions.getGitHubHost(url); + check(ghHost.isGitHubUrl(url), "'%s' is not a valid GitHub url", url); try { credentialHandler = getCredentialHandler( - GITHUB_COM.getHost(), GITHUB_COM.getProjectNameFromUrl(url), credentials); + ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { throw new EvalException("Cannot parse url", e); } @@ -2305,7 +2313,7 @@ public GitHubPrDestination githubPrDestination( "empty_diff_merge_statuses")), convertSlugToConclusion(allowEmptyDiffCheckSuitesToConclusion), getGeneralConsole(), - GITHUB_COM, + ghHost, credentialHandler), Starlark.isNullOrNone(integrates) ? defaultGitIntegrate @@ -2316,7 +2324,7 @@ public GitHubPrDestination githubPrDestination( mainConfigFile, apiCheckerObj != null ? apiCheckerObj : checkerObj, updateDescription, - GITHUB_COM, + ghHost, primaryBranchMigrationMode, checkerObj, credentialHandler); @@ -2962,13 +2970,14 @@ public EndpointProvider githubApi( Checker checker = convertFromNoneable(checkerObj, null); validateEndpointChecker(checker, GITHUB_API); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = gitHubOptions.getGitHubHost(url); CredentialFileHandler credentialHandler = getCredentialHandler(url, credentials); return EndpointProvider.wrap( new GitHubEndPoint( - gitHubOptions.newGitHubApiSupplier(cleanedUrl, checker, credentialHandler, GITHUB_COM), + gitHubOptions.newGitHubApiSupplier(cleanedUrl, checker, credentialHandler, ghHost), cleanedUrl, getGeneralConsole(), - GITHUB_COM, credentialHandler)); + ghHost, credentialHandler)); } @SuppressWarnings("unused") @@ -3158,19 +3167,20 @@ public GitHubTrigger gitHubTrigger( ImmutableSet parsedEvents = handleEventTypes(events, eventBuilder, types); validateEndpointChecker(checker, GITHUB_TRIGGER); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = gitHubOptions.getGitHubHost(url); CredentialFileHandler credentialHandler; try { credentialHandler = getCredentialHandler( - GITHUB_COM.getHost(), GITHUB_COM.getProjectNameFromUrl(url), credentials); + ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { throw new EvalException("Cannot parse url", e); } return new GitHubTrigger( - gitHubOptions.newGitHubApiSupplier(url, checker, credentialHandler, GITHUB_COM), + gitHubOptions.newGitHubApiSupplier(url, checker, credentialHandler, ghHost), url, parsedEvents, getGeneralConsole(), - GITHUB_COM, + ghHost, credentialHandler); } @@ -3389,28 +3399,29 @@ private String fixHttp(String url, Location location) { /** Do not use this for github origins */ protected ApprovalsProvider approvalsProvider(String url) { - Preconditions.checkArgument( - !GITHUB_COM.isGitHubUrl(url), - "Git origins with github should use github approval providers!"); + Preconditions.checkArgument( + !options.get(GitHubOptions.class).isGithubUrl(url), + "Git origins with github should use github approval providers!"); return options.get(GitOriginOptions.class).approvalsProvider; } - protected ApprovalsProvider githubPreSubmitApprovalsProvider( - String url, CredentialFileHandler creds) { + protected ApprovalsProvider githubPreSubmitApprovalsProvider ( + String url, CredentialFileHandler creds) throws EvalException { GeneralOptions generalOptions = options.get(GeneralOptions.class); GitHubOptions githubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = githubOptions.getGitHubHost(url); return new GitHubPreSubmitApprovalsProvider( githubOptions, - GITHUB_COM, + ghHost, new GitHubSecuritySettingsValidator( - githubOptions.newGitHubApiSupplier(url, null, creds, GITHUB_COM), + githubOptions.newGitHubApiSupplier(url, null, creds, ghHost), ImmutableList.copyOf(githubOptions.allStarAppIds), generalOptions.console()), new GitHubUserApprovalsValidator( - githubOptions.newGitHubApiSupplier(url, null, creds, GITHUB_COM), - githubOptions.newGitHubGraphQLApiSupplier(url, null, creds, GITHUB_COM), + githubOptions.newGitHubApiSupplier(url, null, creds, ghHost), + githubOptions.newGitHubGraphQLApiSupplier(url, null, creds, ghHost), generalOptions.console(), - GITHUB_COM, + ghHost, new GetCommitHistoryParams( /* commits= */ githubOptions.gqlOverride.get(0), /* pullRequests= */ githubOptions.gqlOverride.get(1), @@ -3419,21 +3430,22 @@ protected ApprovalsProvider githubPreSubmitApprovalsProvider( } protected ApprovalsProvider githubPostSubmitApprovalsProvider( - String url, String branch, CredentialFileHandler creds) { + String url, String branch, CredentialFileHandler creds) throws EvalException { GeneralOptions generalOptions = options.get(GeneralOptions.class); GitHubOptions githubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = githubOptions.getGitHubHost(url); return new GitHubPostSubmitApprovalsProvider( - GITHUB_COM, + ghHost, branch, new GitHubSecuritySettingsValidator( - githubOptions.newGitHubApiSupplier(url, null, creds, GITHUB_COM), + githubOptions.newGitHubApiSupplier(url, null, creds, ghHost), ImmutableList.copyOf(githubOptions.allStarAppIds), generalOptions.console()), new GitHubUserApprovalsValidator( - githubOptions.newGitHubApiSupplier(url, null, creds, GITHUB_COM), - githubOptions.newGitHubGraphQLApiSupplier(url, null, creds, GITHUB_COM), + githubOptions.newGitHubApiSupplier(url, null, creds, ghHost), + githubOptions.newGitHubGraphQLApiSupplier(url, null, creds, ghHost), generalOptions.console(), - GITHUB_COM, + ghHost, new GetCommitHistoryParams( /* commits= */ githubOptions.gqlOverride.get(0), /* pullRequests= */ githubOptions.gqlOverride.get(1), @@ -3505,9 +3517,15 @@ protected LazyResourceLoader> maybeGetGerritApi( protected LazyResourceLoader> maybeGetGitHubApi( String url, @Nullable Checker checker, @Nullable CredentialFileHandler creds, StarlarkThread thread) { - if (!GITHUB_COM.isGitHubUrl(url)) { - return null; - } + try { + GitHubOptions githubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = githubOptions.getGitHubHost(url); + if (!ghHost.isGitHubUrl(url)) { + return null; + } + } catch (EvalException e) { + return null; + } return (console) -> { try { return githubApi(url, checker, creds, thread); @@ -3535,9 +3553,15 @@ protected LazyResourceLoader> maybeGetGitHubApi( @Nullable protected CredentialFileHandler getCredentialHandler( String url, @Nullable Object starlarkValue) { try { - if (GITHUB_COM.isGitHubUrl(url)) { - url = GITHUB_COM.normalizeUrl(url); - } + try { + GitHubOptions githubOptions = options.get(GitHubOptions.class); + GitHubHost ghHost = githubOptions.getGitHubHost(url); + if (ghHost.isGitHubUrl(url)) { + url = ghHost.normalizeUrl(url); + } + } catch (EvalException e) { + // nothing to-do, it is valid that this is not an GitHub URL. + } URI uri = URI.create(url); return getCredentialHandler(uri.getHost(), uri.getPath(), starlarkValue); } catch (ValidationException | IllegalArgumentException parseEx) { diff --git a/java/com/google/copybara/git/GitRepoType.java b/java/com/google/copybara/git/GitRepoType.java index 305c558fc..4cf61eeb3 100644 --- a/java/com/google/copybara/git/GitRepoType.java +++ b/java/com/google/copybara/git/GitRepoType.java @@ -183,14 +183,14 @@ GitRevision resolveRef( protected static GitRevision maybeFetchGithubPullRequest(GitRepository repository, String repoUrl, String ref, boolean describeVersion, boolean partialFetch) throws RepoException, ValidationException { - // TODO(malcon): This only supports github.com PRs, not enterprise. - Optional githubPrUrl = GitHubHost.GITHUB_COM.maybeParseGithubPrUrl(ref); + GitHubHost ghHost = GitHubHost.fromUrl(repoUrl); + Optional githubPrUrl = ghHost.maybeParseGithubPrUrl(ref); if (githubPrUrl.isPresent()) { // TODO(malcon): Support merge ref too once we have github pr origin. String stableRef = GitHubUtil.asHeadRef(githubPrUrl.get().getPrNumber()); GitRevision gitRevision = repository.fetchSingleRefWithTags( - "https://github.com/" + githubPrUrl.get().getProject(), + ghHost.getHostUrl() + githubPrUrl.get().getProject(), stableRef, /* fetchTags= */ describeVersion, partialFetch, diff --git a/java/com/google/copybara/git/Mirror.java b/java/com/google/copybara/git/Mirror.java index 5f8671cce..7175178ee 100644 --- a/java/com/google/copybara/git/Mirror.java +++ b/java/com/google/copybara/git/Mirror.java @@ -37,6 +37,7 @@ import com.google.copybara.exception.EmptyChangeException; import com.google.copybara.exception.RepoException; import com.google.copybara.exception.ValidationException; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.monitor.EventMonitor.ChangeMigrationFinishedEvent; import com.google.copybara.profiler.Profiler; import com.google.copybara.profiler.Profiler.ProfilerTask; @@ -248,7 +249,8 @@ private void maybeConfigureGitNameAndEmail(GitRepository repo) throws RepoExcept private static String getOriginDestinationRef(String url) throws ValidationException { // TODO(copybara-team): This is used just for normalization. We should be able to do it without // knowing the host. - return GITHUB_COM.isGitHubUrl(url) ? GITHUB_COM.normalizeUrl(url) : url; + GitHubHost host = GitHubHost.fromUrl(url); + return host.isGitHubUrl(url) ? host.normalizeUrl(url) : url; } @VisibleForTesting diff --git a/java/com/google/copybara/git/github/BUILD b/java/com/google/copybara/git/github/BUILD index 89ba6f782..6d172823b 100644 --- a/java/com/google/copybara/git/github/BUILD +++ b/java/com/google/copybara/git/github/BUILD @@ -42,6 +42,7 @@ java_library( ), javacopts = JAVACOPTS, deps = [ + ":util", "//java/com/google/copybara/checks", "//java/com/google/copybara/doc:annotations", # unuseddeps:keep "//java/com/google/copybara/exception", diff --git a/java/com/google/copybara/git/github/api/GitHubApiTransportImpl.java b/java/com/google/copybara/git/github/api/GitHubApiTransportImpl.java index 69b37285f..294d4e2ec 100644 --- a/java/com/google/copybara/git/github/api/GitHubApiTransportImpl.java +++ b/java/com/google/copybara/git/github/api/GitHubApiTransportImpl.java @@ -35,6 +35,7 @@ import com.google.copybara.exception.ValidationException; import com.google.copybara.git.GitCredential.UserPassword; import com.google.copybara.git.GitRepository; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.json.GsonParserUtil; import com.google.copybara.util.console.Console; import java.io.IOException; @@ -55,22 +56,24 @@ public class GitHubApiTransportImpl implements GitHubApiTransport { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final JsonFactory JSON_FACTORY = new GsonFactory(); - private static final String API_URL = "https://api.github.com"; - private static final String GITHUB_WEB_URL = "https://github.com"; private final GitRepository repo; private final HttpTransport httpTransport; private final String storePath; private final Console console; private final boolean bearerAuth; + private final String apiUrl; + private final String githubWebUrl; - public GitHubApiTransportImpl(GitRepository repo, HttpTransport httpTransport, + public GitHubApiTransportImpl(GitHubHost ghHost, GitRepository repo, HttpTransport httpTransport, String storePath, boolean bearerAuth, Console console) { this.repo = Preconditions.checkNotNull(repo); this.httpTransport = Preconditions.checkNotNull(httpTransport); this.storePath = storePath; this.console = Preconditions.checkNotNull(console); this.bearerAuth = bearerAuth; + this.apiUrl = ghHost.getAPIEndpoint(); + this.githubWebUrl = ghHost.getHostUrl(); } @SuppressWarnings("unchecked") @@ -87,7 +90,7 @@ public T get(String path, Type responseType, ImmutableListMultimap T post(String path, Object request, Type responseType, String request response, responseType, false); if (responseObj instanceof PaginatedPayload) { return (T) - ((PaginatedPayload) responseObj).annotatePayload(API_URL, maybeGetLinkHeader(response)); + ((PaginatedPayload) responseObj).annotatePayload(apiUrl, maybeGetLinkHeader(response)); } return (T) responseObj; @@ -221,34 +224,34 @@ private HttpRequestFactory getHttpRequestFactory( private GenericUrl getFullEndpointURL(String path) { String maybePrefix = path.startsWith("/") ? "" : "/"; - return new GenericUrl(URI.create(API_URL + maybePrefix + path)); + return new GenericUrl(URI.create(apiUrl + maybePrefix + path)); } /** * Gets the credentials from git credential helper. First we try - * to get it for the api.github.com host, just in case the user has an specific token for that - * url, otherwise we use the github.com host one. + * to get it for the API endpoint host, just in case the user has an specific token for that + * url, otherwise we use the web url host one. */ private UserPassword getCredentials() throws RepoException, ValidationException { try { - return repo.credentialFill(API_URL); + return repo.credentialFill(apiUrl); } catch (ValidationException e) { try { - return repo.credentialFill(GITHUB_WEB_URL); + return repo.credentialFill(githubWebUrl); } catch (ValidationException e1) { // Ugly, but helpful... throw new ValidationException(String.format( - "Cannot get credentials for host https://api.github.com or https://github.com from" + "Cannot get credentials for host %s or %s from" + " credentials helper. Make sure either your credential helper has the username" + " and password/token or if you don't use one, that file '%s'" + " contains one of the two lines: \nEither:\n" - + "https://USERNAME:TOKEN@api.github.com\n" + + "https://USERNAME:TOKEN@%s\n" + "or:\n" - + "https://USERNAME:TOKEN@github.com\n" + + "https://USERNAME:TOKEN@%s\n" + "\n" + "Note that spaces or other special characters need to be escaped. For example" + " ' ' should be %%20 and '@' should be %%40 (For example when using the email" - + " as username)", storePath), e1); + + " as username)", apiUrl, githubWebUrl, apiUrl, githubWebUrl, storePath), e1); } } } diff --git a/java/com/google/copybara/git/github/util/GitHubHost.java b/java/com/google/copybara/git/github/util/GitHubHost.java index f2b55f0ce..c84236aba 100644 --- a/java/com/google/copybara/git/github/util/GitHubHost.java +++ b/java/com/google/copybara/git/github/util/GitHubHost.java @@ -41,6 +41,12 @@ public GitHubHost(String host) { this.gitHubPrUrlPattern = Pattern.compile("https://\\Q" + host + "\\E/(.+)/pull/([0-9]+)"); } + static public GitHubHost fromUrl(String url) + { + url = url.replaceAll("http\\:\\/\\/|https\\:\\/\\/|git\\+|git@", "").replaceAll(":.*|/.*", ""); + return new GitHubHost(url); + } + /** * Return the username part of a github url. For example in https://github.com/foo/bar/baz, 'foo' * would be the user. @@ -102,6 +108,10 @@ public String getHost() { return host; } + public String getHostUrl() { + return "https://" + host + "/"; + } + public String normalizeUrl(String url) throws ValidationException { return projectAsUrl(getProjectNameFromUrl(url)); } @@ -113,6 +123,17 @@ public Optional maybeParseGithubPrUrl(String ref) { : Optional.empty(); } + public String getAPIEndpoint() + { + if(host.contains("github.com")) + { + return "https://api.github.com"; + } else { + // https://docs.github.com/en/enterprise-server@3.16/rest/enterprise-admin?apiVersion=2022-11-28 + return "https://" + host + "/api/v3"; + } + } + /** A GitHub PR project and number */ public static class GitHubPrUrl { diff --git a/java/com/google/copybara/rust/RustModule.java b/java/com/google/copybara/rust/RustModule.java index 04b0f0df4..99c4a9f9f 100644 --- a/java/com/google/copybara/rust/RustModule.java +++ b/java/com/google/copybara/rust/RustModule.java @@ -456,8 +456,8 @@ protected String getFuzzersDownloadUrl(Path cargoTomlPath) } private static String normalizeUrl(String url) throws ValidationException { - if (GitHubHost.GITHUB_COM.isGitHubUrl(url)) { - url = GitHubHost.GITHUB_COM.normalizeUrl(url); + if (GitHubHost.fromUrl(url).isGitHubUrl(url)) { + url = GitHubHost.fromUrl(url).normalizeUrl(url); } return url; } diff --git a/javatests/com/google/copybara/git/GitOriginTest.java b/javatests/com/google/copybara/git/GitOriginTest.java index b0954197b..3e0aec18a 100644 --- a/javatests/com/google/copybara/git/GitOriginTest.java +++ b/javatests/com/google/copybara/git/GitOriginTest.java @@ -336,6 +336,24 @@ public void testGithubOrigin() throws Exception { + "}"); } + @Test + public void testGithubOriginForEnterpriseUrl() throws Exception { + options.github.gitHubAllowedHosts = ImmutableList.of("some.github-enterprise.net"); + origin = skylark.eval("result", + "result = git.github_origin(\n" + + " url = 'https://some.github-enterprise.net/copybara',\n" + + " ref = 'main',\n" + + ")"); + assertThat(origin.toString()) + .isEqualTo( + "GitOrigin{" + + "repoUrl=https://some.github-enterprise.net/copybara, " + + "ref=main, " + + "repoType=GITHUB, " + + "primaryBranchMigrationMode=false" + + "}"); + } + @Test public void testInvalidGithubUrl() throws Exception { ValidationException expected = @@ -350,7 +368,7 @@ public void testInvalidGithubUrl() throws Exception { + ")")); console .assertThat() - .onceInLog(MessageType.ERROR, ".*Invalid Github URL: https://foo.com/copybara.*"); + .onceInLog(MessageType.ERROR, ".*'foo.com' is not a valid GitHub url.*"); } @Test @@ -367,7 +385,7 @@ public void testInvalidGithubUrlWithGithubString() throws Exception { + ")")); console .assertThat() - .onceInLog(MessageType.ERROR, ".*Invalid Github URL: https://foo.com/github.com.*"); + .onceInLog(MessageType.ERROR, ".*'foo.com' is not a valid GitHub url.*"); } @Test diff --git a/javatests/com/google/copybara/git/github/api/GitHubApiTest.java b/javatests/com/google/copybara/git/github/api/GitHubApiTest.java index a08696019..86b01e36e 100644 --- a/javatests/com/google/copybara/git/github/api/GitHubApiTest.java +++ b/javatests/com/google/copybara/git/github/api/GitHubApiTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import com.google.copybara.git.GitRepository; import com.google.copybara.git.github.api.testing.AbstractGitHubApiTest; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.util.console.testing.TestingConsole; import java.io.IOException; import java.net.URI; @@ -110,7 +111,7 @@ public LowLevelHttpResponse execute() throws IOException { return request; } }; - return new GitHubApiTransportImpl( + return new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "some_storage_file", false, new TestingConsole()); } diff --git a/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java b/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java index 409c7b5ed..4df92a688 100644 --- a/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java +++ b/javatests/com/google/copybara/git/github/api/GitHubApiTransportImplTest.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableList; import com.google.copybara.exception.RepoException; import com.google.copybara.git.GitRepository; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.util.console.testing.TestingConsole; import java.io.IOException; import java.nio.file.Files; @@ -104,7 +105,7 @@ public LowLevelHttpResponse execute() throws IOException { }; } }; - transport = new GitHubApiTransportImpl( + transport = new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "store", false, new TestingConsole()); String unused = transport.get(String.class, "foo/bar"); assertThat(headers).containsEntry("authorization", ImmutableList.of("Basic dXNlcjpTRUNSRVQ=")); @@ -127,7 +128,7 @@ public LowLevelHttpResponse execute() throws IOException { }; } }; - transport = new GitHubApiTransportImpl( + transport = new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "store", true, new TestingConsole()); String unused = transport.get(String.class, "foo/bar"); assertThat(headers).containsEntry("authorization", ImmutableList.of("Bearer SECRET")); @@ -137,7 +138,7 @@ private void runTestThrowsHttpResponseException(Callable c) throws Exception HttpResponseException ex = new HttpResponseException.Builder(STATUS_CODE, ERROR_MESSAGE, new HttpHeaders()).build(); httpTransport = createMockHttpTransport(ex); - transport = new GitHubApiTransportImpl( + transport = new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "store", false, new TestingConsole()); try { c.call(); @@ -151,7 +152,7 @@ private void runTestThrowsHttpResponseException(Callable c) throws Exception private void runTestThrowsIOException(Callable c) throws Exception { IOException ioException = new IOException(); httpTransport = createMockHttpTransport(ioException); - transport = new GitHubApiTransportImpl( + transport = new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "store", false, new TestingConsole()); try { c.call(); diff --git a/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java b/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java index 351ee2779..63c9a2f96 100644 --- a/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java +++ b/javatests/com/google/copybara/git/github/api/GitHubGraphQLTest.java @@ -29,6 +29,7 @@ import com.google.api.client.testing.http.MockLowLevelHttpResponse; import com.google.copybara.git.GitRepository; import com.google.copybara.git.github.api.testing.AbstractGitHubGraphQLApiTest; +import com.google.copybara.git.github.util.GitHubHost; import com.google.copybara.util.console.testing.TestingConsole; import java.io.IOException; import java.nio.file.Files; @@ -105,7 +106,7 @@ public LowLevelHttpResponse execute() throws IOException { return request; } }; - return new GitHubApiTransportImpl( + return new GitHubApiTransportImpl(GitHubHost.GITHUB_COM, repo, httpTransport, "some_storage_file", false, new TestingConsole()); } From ea48ff414b86098e26b5db515b6a108641e2cbfc Mon Sep 17 00:00:00 2001 From: Chris Campos Date: Fri, 25 Jul 2025 11:04:47 -0400 Subject: [PATCH 2/2] Allow GitHub origin to accept any host To better support GitHub enterprise, the GitHub Origin will just assume any host it's given is a GitHub or Github enterprise host. GitOrigin will continue to accept GitHub.com URLs. --- docs/reference.md | 8 +- .../google/copybara/git/GitHubOptions.java | 65 +++++++++---- java/com/google/copybara/git/GitModule.java | 96 ++++++++++++------- java/com/google/copybara/git/GitRepoType.java | 9 +- java/com/google/copybara/git/Mirror.java | 3 +- .../copybara/git/github/util/GitHubHost.java | 55 +++++++---- java/com/google/copybara/rust/RustModule.java | 2 +- .../copybara/git/GitHubPrDestinationTest.java | 4 +- .../google/copybara/git/GitOriginTest.java | 35 ------- 9 files changed, 154 insertions(+), 123 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 49a4f50ed..e823cb789 100755 --- a/docs/reference.md +++ b/docs/reference.md @@ -3394,7 +3394,6 @@ Parameter | Description Name | Type | Description ---- | ---- | ----------- `--allstar-app-ids` | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3443,7 +3442,6 @@ Name | Type | Description `--git-destination-push` | *string* | If set, overrides the git destination push reference. `--git-destination-url` | *string* | If set, overrides the git destination URL. `--git-skip-checker` | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3452,7 +3450,7 @@ Name | Type | Description ### git.github_origin -Defines a Git origin for a Github repository. This origin should be used for public branches. Use github_pr_origin for importing Pull Requests. +Defines a Git origin for a GitHub or GitHub Enterprise repository. This origin should be used for public branches. Use github_pr_origin for importing Pull Requests. origin git.github_origin(url, ref=None, submodules='NO', excluded_submodules=[], first_parent=True, partial_fetch=False, patch=None, describe_version=None, version_selector=None, primary_branch_migration=False, enable_lfs=False, credentials=None) @@ -3485,7 +3483,6 @@ Name | Type | Description `--git-origin-log-batch` | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB. `--git-origin-non-linear-history` | *boolean* | Read the full git log and skip changes before the from ref rather than using a log path. `--git-origin-rebase-ref` | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR @@ -3581,7 +3578,6 @@ Name | Type | Description `--git-destination-push` | *string* | If set, overrides the git destination push reference. `--git-destination-url` | *string* | If set, overrides the git destination URL. `--git-skip-checker` | *boolean* | If true and git.destination has a configured checker, it will not be used in the migration. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--github-destination-pr-branch` | *string* | If set, uses this branch for creating the pull request instead of using a generated one @@ -3649,7 +3645,6 @@ Name | Type | Description `--git-origin-log-batch` | *int* | Read the origin git log in batches of n commits. Might be needed for large migrations resulting in git logs of more than 1 GB. `--git-origin-non-linear-history` | *boolean* | Read the full git log and skip changes before the from ref rather than using a log path. `--git-origin-rebase-ref` | *string* | When importing a change from a Git origin ref, it will be rebased to this ref, if set. A common use case: importing a Github PR, rebase it to the main branch (usually 'master'). Note that, if the repo uses submodules, they won't be rebased. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--github-force-import` | *boolean* | Force import regardless of the state of the PR @@ -3689,7 +3684,6 @@ Parameter | Description Name | Type | Description ---- | ---- | ----------- `--allstar-app-ids` | *list* | Flag used to set AllStar GitHub app id aliases. See https://github.com/ossf/allstar. -`--github-allowed-hosts` | *list* | If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported. `--github-api-bearer-auth` | *boolean* | If using a token for GitHub access, bearer auth might be required `--github-destination-delete-pr-branch` | *boolean* | Overwrite git.github_destination delete_pr_branch field `--gql-commit-history-override` | *list* | Flag used to target GraphQL params 'first' arguments in the event the defaults are over or underusing the api ratelimit. The flag value should be semicolon separated. This should be rarely used for repos that don't fit well in our defaults. E.g. '50;5;5' represent 50 commits, 5 PRs for each commit, 5 reviews per PR diff --git a/java/com/google/copybara/git/GitHubOptions.java b/java/com/google/copybara/git/GitHubOptions.java index cc567e62e..0ff4c9998 100644 --- a/java/com/google/copybara/git/GitHubOptions.java +++ b/java/com/google/copybara/git/GitHubOptions.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; import com.google.copybara.GeneralOptions; import com.google.copybara.LazyResourceLoader; import com.google.copybara.Option; @@ -41,6 +42,7 @@ import com.google.copybara.jcommander.SemicolonSeparatedListSplitter; import com.google.copybara.starlark.StarlarkUtil; import com.google.copybara.util.console.Console; +import java.util.Set; import net.starlark.java.eval.EvalException; import java.io.IOException; @@ -89,26 +91,8 @@ public class GitHubOptions implements Option { arity = 1) public boolean gitHubApiBearerAuth = false; - @Parameter( - names = "--github-allowed-hosts", - description = "If using GitHub Enterprise, one needs to specify valid hosts. By default only `github.com` is supported." - ) - public List gitHubAllowedHosts = ImmutableList.of("github.com"); - - public GitHubHost getGitHubHost(String url) throws EvalException { - GitHubHost host = GitHubHost.fromUrl(url); - StarlarkUtil.check(gitHubAllowedHosts.contains(host.getHost()), "'%s' is not a valid GitHub url", host.getHost()); - return host; - } - - public boolean isGithubUrl(String url) - { - GitHubHost host = GitHubHost.fromUrl(url); - if(gitHubAllowedHosts.contains(host.getHost())){ - return host.isGitHubUrl(url); - } - - return false; + public GitHubHost getGitHubHost(String url) throws ValidationException { + return GitHubHost.fromUrl(url); } public GitHubOptions(GeneralOptions generalOptions, GitOptions gitOptions) { @@ -151,6 +135,26 @@ public GitHubApi newGitHubRestApi(GitHubHost ghHost, gitHubProject, /* checker= */ null, credentials, generalOptions.console()); } + /** + * Returns a new Github.com specific {@link GitHubApi} instance for the given project enforcing the given {@link + * Checker}. + * + *

The project for 'https://github.com/foo/bar' is 'foo/bar'. + * + * @param gitHubProject the project + * @param checker the checker to enforce + * @param credentials the credentials to use for GitHub API auth + * @param console the console, used for logging + * @return the instance + * @throws RepoException if there is a failure in using the credentials + */ + public GitHubApi newGitHubRestApi(String gitHubProject, + @Nullable Checker checker, + @Nullable CredentialFileHandler credentials, + Console console) throws RepoException { + return newGitHubRestApi(GitHubHost.GITHUB_COM, gitHubProject, checker, credentials, console); + } + /** * Returns a new {@link GitHubApi} instance for the given project enforcing the given {@link * Checker}. @@ -187,6 +191,27 @@ public GitHubGraphQLApi newGitHubGraphQLApi(GitHubHost ghHost, gitHubProject, /* checker= */ null, credentials, generalOptions.console()); } + /** + * Returns a new GitHub.com specific {@link GitHubApi} instance for the given project enforcing the given {@link + * Checker}. + * + *

The project for 'https://github.com/foo/bar' is 'foo/bar'. + * @param gitHubProject the GitHub project + * @param checker the checker to enforce + * @param credentials the credentials to use for the GitHub API + * @param console the console, for logging + * @return the instance + * @throws RepoException if there is an issue using the provided credentials + */ + public GitHubGraphQLApi newGitHubGraphQLApi( + String gitHubProject, + @Nullable Checker checker, + @Nullable CredentialFileHandler credentials, + Console console) + throws RepoException { + return newGitHubGraphQLApi(GitHubHost.GITHUB_COM, gitHubProject, checker, credentials, console); + } + /** * Returns a new {@link GitHubApi} instance for the given project enforcing the given {@link * Checker}. diff --git a/java/com/google/copybara/git/GitModule.java b/java/com/google/copybara/git/GitModule.java index 385a015e0..8f84cf382 100644 --- a/java/com/google/copybara/git/GitModule.java +++ b/java/com/google/copybara/git/GitModule.java @@ -35,7 +35,6 @@ import static com.google.copybara.git.GitHubPrOrigin.GITHUB_PR_USE_MERGE; import static com.google.copybara.git.GitOptions.USE_CREDENTIALS_FROM_CONFIG; import static com.google.copybara.git.github.api.GitHubEventType.WATCHABLE_EVENTS; -import static com.google.copybara.git.github.util.GitHubHost.GITHUB_COM; import static com.google.copybara.version.LatestVersionSelector.VersionElementType.ALPHABETIC; import static com.google.copybara.version.LatestVersionSelector.VersionElementType.NUMERIC; import static java.util.Arrays.stream; @@ -356,7 +355,8 @@ public GitOrigin origin( CredentialFileHandler credentialHandler = getCredentialHandler(fixedUrl, credentials); GitHubOptions githubOptions = options.get(GitHubOptions.class); - boolean isGitHubUrl = githubOptions.isGithubUrl(url); + // This does not support GitHub Enterprise. For that, use githubOrigin. + boolean isGitHubUrl = GitHubHost.isGitHubUrl(url); return GitOrigin.newGitOrigin( options, @@ -1163,7 +1163,12 @@ public Origin githubPrOrigin( throws EvalException { checkNotEmpty(url, "url"); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = gitHubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = gitHubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new EvalException(e); + } PatchTransformation patchTransformation = maybeGetPatchTransformation(patch); List excludedSubmoduleList = @@ -1263,7 +1268,7 @@ public Origin githubPrOrigin( @StarlarkMethod( name = "github_origin", doc = - "Defines a Git origin for a Github repository. This origin should be used for public" + "Defines a Git origin for a GitHub or GitHub Enterprise repository. This origin should be used for public" + " branches. Use " + GITHUB_PR_ORIGIN_NAME + " for importing Pull Requests.", @@ -1394,7 +1399,11 @@ public GitOrigin githubOrigin( StarlarkThread thread) throws EvalException { GitHubOptions gitHubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = gitHubOptions.getGitHubHost(checkNotEmpty(url, "url")); + try { + gitHubOptions.getGitHubHost(checkNotEmpty(url, "url")); + } catch (ValidationException e) { + throw new EvalException(e); + } if (versionSelector != Starlark.NONE) { check( @@ -1973,7 +1982,12 @@ public GitDestination gitHubDestination( branchToUpdate != null || deletePrBranch == null, "'delete_pr_branch' can only be set if 'pr_branch_to_update' is used"); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = gitHubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = gitHubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new EvalException(e); + } WorkflowOptions workflowOptions = options.get(WorkflowOptions.class); String effectivePrBranchToUpdate = branchToUpdate; @@ -1997,7 +2011,7 @@ public GitDestination gitHubDestination( credentialHandler = getCredentialHandler( ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { - throw new EvalException("Cannot parse url", e); + throw new EvalException(String.format("Cannot parse url '%s'", url), e); } return new GitDestination( repoUrl, @@ -2280,13 +2294,17 @@ public GitHubPrDestination githubPrDestination( Checker apiCheckerObj = convertFromNoneable(apiChecker, null); Checker checkerObj = convertFromNoneable(checker, null); CredentialFileHandler credentialHandler; - GitHubHost ghHost = gitHubOptions.getGitHubHost(url); - check(ghHost.isGitHubUrl(url), "'%s' is not a valid GitHub url", url); + GitHubHost ghHost; + try { + ghHost = gitHubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new EvalException(e); + } try { credentialHandler = getCredentialHandler( ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { - throw new EvalException("Cannot parse url", e); + throw new EvalException(String.format("Cannot parse url '%s'", url), e); } return new GitHubPrDestination( fixHttp( @@ -2970,7 +2988,12 @@ public EndpointProvider githubApi( Checker checker = convertFromNoneable(checkerObj, null); validateEndpointChecker(checker, GITHUB_API); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = gitHubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = gitHubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new EvalException(e); + } CredentialFileHandler credentialHandler = getCredentialHandler(url, credentials); return EndpointProvider.wrap( new GitHubEndPoint( @@ -3167,13 +3190,18 @@ public GitHubTrigger gitHubTrigger( ImmutableSet parsedEvents = handleEventTypes(events, eventBuilder, types); validateEndpointChecker(checker, GITHUB_TRIGGER); GitHubOptions gitHubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = gitHubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = gitHubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new EvalException(e); + } CredentialFileHandler credentialHandler; try { credentialHandler = getCredentialHandler( ghHost.getHost(), ghHost.getProjectNameFromUrl(url), credentials); } catch (ValidationException e) { - throw new EvalException("Cannot parse url", e); + throw new EvalException(String.format("Cannot parse url '%s'", url), e); } return new GitHubTrigger( gitHubOptions.newGitHubApiSupplier(url, checker, credentialHandler, ghHost), @@ -3400,16 +3428,21 @@ private String fixHttp(String url, Location location) { /** Do not use this for github origins */ protected ApprovalsProvider approvalsProvider(String url) { Preconditions.checkArgument( - !options.get(GitHubOptions.class).isGithubUrl(url), + !GitHubHost.isGitHubUrl(url), "Git origins with github should use github approval providers!"); return options.get(GitOriginOptions.class).approvalsProvider; } protected ApprovalsProvider githubPreSubmitApprovalsProvider ( - String url, CredentialFileHandler creds) throws EvalException { + String url, CredentialFileHandler creds) { GeneralOptions generalOptions = options.get(GeneralOptions.class); GitHubOptions githubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = githubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = githubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new IllegalStateException(e); + } return new GitHubPreSubmitApprovalsProvider( githubOptions, ghHost, @@ -3430,10 +3463,15 @@ protected ApprovalsProvider githubPreSubmitApprovalsProvider ( } protected ApprovalsProvider githubPostSubmitApprovalsProvider( - String url, String branch, CredentialFileHandler creds) throws EvalException { + String url, String branch, CredentialFileHandler creds) { GeneralOptions generalOptions = options.get(GeneralOptions.class); GitHubOptions githubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = githubOptions.getGitHubHost(url); + GitHubHost ghHost; + try { + ghHost = githubOptions.getGitHubHost(url); + } catch (ValidationException e) { + throw new IllegalStateException(e); + } return new GitHubPostSubmitApprovalsProvider( ghHost, branch, @@ -3517,13 +3555,8 @@ protected LazyResourceLoader> maybeGetGerritApi( protected LazyResourceLoader> maybeGetGitHubApi( String url, @Nullable Checker checker, @Nullable CredentialFileHandler creds, StarlarkThread thread) { - try { - GitHubOptions githubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = githubOptions.getGitHubHost(url); - if (!ghHost.isGitHubUrl(url)) { - return null; - } - } catch (EvalException e) { + GitHubOptions githubOptions = options.get(GitHubOptions.class); + if (!GitHubHost.isGitHubUrl(url)) { return null; } return (console) -> { @@ -3553,15 +3586,10 @@ protected LazyResourceLoader> maybeGetGitHubApi( @Nullable protected CredentialFileHandler getCredentialHandler( String url, @Nullable Object starlarkValue) { try { - try { - GitHubOptions githubOptions = options.get(GitHubOptions.class); - GitHubHost ghHost = githubOptions.getGitHubHost(url); - if (ghHost.isGitHubUrl(url)) { - url = ghHost.normalizeUrl(url); - } - } catch (EvalException e) { - // nothing to-do, it is valid that this is not an GitHub URL. - } + GitHubOptions githubOptions = options.get(GitHubOptions.class); + if (GitHubHost.isGitHubUrl(url)) { + url = githubOptions.getGitHubHost(url).normalizeUrl(url); + } URI uri = URI.create(url); return getCredentialHandler(uri.getHost(), uri.getPath(), starlarkValue); } catch (ValidationException | IllegalArgumentException parseEx) { diff --git a/java/com/google/copybara/git/GitRepoType.java b/java/com/google/copybara/git/GitRepoType.java index 4cf61eeb3..078ea8742 100644 --- a/java/com/google/copybara/git/GitRepoType.java +++ b/java/com/google/copybara/git/GitRepoType.java @@ -183,14 +183,17 @@ GitRevision resolveRef( protected static GitRevision maybeFetchGithubPullRequest(GitRepository repository, String repoUrl, String ref, boolean describeVersion, boolean partialFetch) throws RepoException, ValidationException { - GitHubHost ghHost = GitHubHost.fromUrl(repoUrl); - Optional githubPrUrl = ghHost.maybeParseGithubPrUrl(ref); + if (GitHubHost.isGitHubUrl(repoUrl)) { + + } + Optional ghHost = Optional.ofNullable(GitHubHost.isGitHubUrl(repoUrl) ? GitHubHost.fromUrl(repoUrl) : null); + Optional githubPrUrl = ghHost.flatMap(host -> host.maybeParseGithubPrUrl(ref)); if (githubPrUrl.isPresent()) { // TODO(malcon): Support merge ref too once we have github pr origin. String stableRef = GitHubUtil.asHeadRef(githubPrUrl.get().getPrNumber()); GitRevision gitRevision = repository.fetchSingleRefWithTags( - ghHost.getHostUrl() + githubPrUrl.get().getProject(), + ghHost.get().getHostUrl() + githubPrUrl.get().getProject(), stableRef, /* fetchTags= */ describeVersion, partialFetch, diff --git a/java/com/google/copybara/git/Mirror.java b/java/com/google/copybara/git/Mirror.java index 7175178ee..2c344d1f2 100644 --- a/java/com/google/copybara/git/Mirror.java +++ b/java/com/google/copybara/git/Mirror.java @@ -249,8 +249,7 @@ private void maybeConfigureGitNameAndEmail(GitRepository repo) throws RepoExcept private static String getOriginDestinationRef(String url) throws ValidationException { // TODO(copybara-team): This is used just for normalization. We should be able to do it without // knowing the host. - GitHubHost host = GitHubHost.fromUrl(url); - return host.isGitHubUrl(url) ? host.normalizeUrl(url) : url; + return GitHubHost.isGitHubUrl(url) ? GitHubHost.fromUrl(url).normalizeUrl(url) : url; } @VisibleForTesting diff --git a/java/com/google/copybara/git/github/util/GitHubHost.java b/java/com/google/copybara/git/github/util/GitHubHost.java index c84236aba..3ff2af500 100644 --- a/java/com/google/copybara/git/github/util/GitHubHost.java +++ b/java/com/google/copybara/git/github/util/GitHubHost.java @@ -21,30 +21,32 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Strings; +import com.google.common.collect.Sets; import com.google.copybara.exception.ValidationException; import com.google.re2j.Matcher; import com.google.re2j.Pattern; import java.net.URI; import java.util.Optional; +import java.util.Set; /** An object that parses GitHub urls in their components (project, name, etc.) */ public class GitHubHost { - + private static final Set KNOWN_GITHUB_HOSTS = Sets.newConcurrentHashSet(); /** Host for http://github.com (Non-Enterprise) */ public static final GitHubHost GITHUB_COM = new GitHubHost("github.com"); - + private static final Pattern GIT_PROTOCOL_MATCHER = Pattern.compile("git@([^:]+):.*"); private final Pattern gitHubPrUrlPattern; private final String host; public GitHubHost(String host) { this.host = checkNotNull(host); this.gitHubPrUrlPattern = Pattern.compile("https://\\Q" + host + "\\E/(.+)/pull/([0-9]+)"); + KNOWN_GITHUB_HOSTS.add(this); } - static public GitHubHost fromUrl(String url) - { - url = url.replaceAll("http\\:\\/\\/|https\\:\\/\\/|git\\+|git@", "").replaceAll(":.*|/.*", ""); - return new GitHubHost(url); + public static GitHubHost fromUrl(String url) throws ValidationException { + URI uri = getUriFromUrl(url); + return new GitHubHost(uri.getHost()); } /** @@ -57,6 +59,27 @@ public String getUserNameFromUrl(String url) throws ValidationException { return i == -1 ? project : project.substring(0, i); } + private static URI getUriFromUrl(String url) throws ValidationException { + Matcher matcher = GIT_PROTOCOL_MATCHER.matcher(url); + if (matcher.matches()) { + String gitProtocolPrefix = "git@" + matcher.group(1) + ":"; + if (url.startsWith(gitProtocolPrefix)) { + url = matcher.group(1) + "/" + url.substring(gitProtocolPrefix.length()).replaceAll("([.]git|/)$", ""); + } + } + + URI uri; + try { + uri = URI.create(url); + } catch (IllegalArgumentException e) { + throw new ValidationException("Cannot find project name from url " + url, e); + } + if (uri.getScheme() == null) { + uri = URI.create("notimportant://" + url); + } + return uri; + } + /** * Given a GitHub host name and a url that represents a GitHub repository, return the project * name, e.g. org/repo. @@ -68,15 +91,7 @@ public String getProjectNameFromUrl(String url) throws ValidationException { if (url.startsWith(gitProtocolPrefix)) { return url.substring(gitProtocolPrefix.length()).replaceAll("([.]git|/)$", ""); } - URI uri; - try { - uri = URI.create(url); - } catch (IllegalArgumentException e) { - throw new ValidationException("Cannot find project name from url " + url, e); - } - if (uri.getScheme() == null) { - uri = URI.create("notimportant://" + url); - } + URI uri = getUriFromUrl(url); checkCondition( host.equals(uri.getHost()), "Not a github url: %s. Expected host: %s", url, host); @@ -90,10 +105,14 @@ public String getProjectNameFromUrl(String url) throws ValidationException { return name; } - /** Returns true if url is a GitHub url for a given GitHub or Enterprise host. */ - public boolean isGitHubUrl(String url) { + /** Returns true if URL belongs to the host that this object is initialized with. */ + public static boolean isGitHubUrl(String url) { + return KNOWN_GITHUB_HOSTS.stream().anyMatch(host -> host.isGitHubUrlForHost(url)); + } + + private boolean isGitHubUrlForHost(String url) { try { - getProjectNameFromUrl(url); + this.getProjectNameFromUrl(url); return true; } catch (ValidationException e) { return false; diff --git a/java/com/google/copybara/rust/RustModule.java b/java/com/google/copybara/rust/RustModule.java index 99c4a9f9f..b72ff3d62 100644 --- a/java/com/google/copybara/rust/RustModule.java +++ b/java/com/google/copybara/rust/RustModule.java @@ -456,7 +456,7 @@ protected String getFuzzersDownloadUrl(Path cargoTomlPath) } private static String normalizeUrl(String url) throws ValidationException { - if (GitHubHost.fromUrl(url).isGitHubUrl(url)) { + if (GitHubHost.isGitHubUrl(url)) { url = GitHubHost.fromUrl(url).normalizeUrl(url); } return url; diff --git a/javatests/com/google/copybara/git/GitHubPrDestinationTest.java b/javatests/com/google/copybara/git/GitHubPrDestinationTest.java index 0f5491e4a..f0385fb89 100644 --- a/javatests/com/google/copybara/git/GitHubPrDestinationTest.java +++ b/javatests/com/google/copybara/git/GitHubPrDestinationTest.java @@ -597,9 +597,7 @@ public void testFindProject() throws ValidationException { ValidationException e = assertThrows( ValidationException.class, () -> checkFindProject("https://github.com", "foo")); - console - .assertThat() - .onceInLog(MessageType.ERROR, ".*'https://github.com' is not a valid GitHub url.*"); + assertThat(e).hasMessageThat().containsMatch(".*Cannot parse url 'https://github.com'.*"); } @Test diff --git a/javatests/com/google/copybara/git/GitOriginTest.java b/javatests/com/google/copybara/git/GitOriginTest.java index 3e0aec18a..8971e96b6 100644 --- a/javatests/com/google/copybara/git/GitOriginTest.java +++ b/javatests/com/google/copybara/git/GitOriginTest.java @@ -338,7 +338,6 @@ public void testGithubOrigin() throws Exception { @Test public void testGithubOriginForEnterpriseUrl() throws Exception { - options.github.gitHubAllowedHosts = ImmutableList.of("some.github-enterprise.net"); origin = skylark.eval("result", "result = git.github_origin(\n" + " url = 'https://some.github-enterprise.net/copybara',\n" @@ -354,40 +353,6 @@ public void testGithubOriginForEnterpriseUrl() throws Exception { + "}"); } - @Test - public void testInvalidGithubUrl() throws Exception { - ValidationException expected = - assertThrows( - ValidationException.class, - () -> - skylark.eval( - "result", - "result = git.github_origin(\n" - + " url = 'https://foo.com/copybara',\n" - + " ref = 'main',\n" - + ")")); - console - .assertThat() - .onceInLog(MessageType.ERROR, ".*'foo.com' is not a valid GitHub url.*"); - } - - @Test - public void testInvalidGithubUrlWithGithubString() throws Exception { - ValidationException expected = - assertThrows( - ValidationException.class, - () -> - skylark.eval( - "result", - "result = git.github_origin(\n" - + " url = 'https://foo.com/github.com',\n" - + " ref = 'main',\n" - + ")")); - console - .assertThat() - .onceInLog(MessageType.ERROR, ".*'foo.com' is not a valid GitHub url.*"); - } - @Test public void testResolveWithUrl() throws Exception { assertThat(origin.resolve(defaultBranch).getUrl()).isEqualTo(url);