Add HTTP-to-CD mirroring

Motivation:

It would be very useful to set up a simple mirror that mirrors the
result of an HTTP query into a repository, just like IFTTT.

Modifications:

- Added support for the following two new URL schemes:
  - `http`
  - `https`
- Renamed `Mirror.remotePath` to `remoteSubpath` to better reflect what
  it actually does.
- Made `Mirror.remoteSubpath` nullable because HTTP-to-CD mirror doesn't
  need it.
- Added `HttpMirror` and `HttpMirrorTest`.

Result:

- A user can mirror a certain HTTP endpoint into a file in Central Dogma.
- Closes line#154

Future works:

- Add support for basic and token-based authentication
- Add documentation

it/build.gradle

@@ -1,4 +1,5 @@
 dependencies {
+ testImplementation libs.armeria.junit5
  // jGit
  testImplementation libs.jgit
  // JSch

it/src/test/java/com/linecorp/centraldogma/it/mirror/http/HttpMirrorTest.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.linecorp.centraldogma.it.mirror.http;
+
+import static com.google.common.base.MoreObjects.firstNonNull;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import javax.annotation.Nullable;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import com.google.common.base.Strings;
+
+import com.linecorp.armeria.common.ContentTooLargeException;
+import com.linecorp.armeria.server.HttpStatusException;
+import com.linecorp.armeria.server.ServerBuilder;
+import com.linecorp.armeria.server.annotation.Get;
+import com.linecorp.armeria.server.annotation.Param;
+import com.linecorp.armeria.server.annotation.StatusCode;
+import com.linecorp.armeria.testing.junit5.server.ServerExtension;
+import com.linecorp.centraldogma.client.CentralDogma;
+import com.linecorp.centraldogma.common.Change;
+import com.linecorp.centraldogma.common.Entry;
+import com.linecorp.centraldogma.common.EntryNotFoundException;
+import com.linecorp.centraldogma.server.CentralDogmaBuilder;
+import com.linecorp.centraldogma.server.MirrorException;
+import com.linecorp.centraldogma.server.MirroringService;
+import com.linecorp.centraldogma.server.storage.project.Project;
+import com.linecorp.centraldogma.testing.internal.TestUtil;
+import com.linecorp.centraldogma.testing.junit.CentralDogmaExtension;
+
+class HttpMirrorTest {
+
+ private static final int MAX_NUM_BYTES = 1024; // 1 KiB
+
+ private static final String REPO_FOO = "foo";
+
+ @RegisterExtension
+ static final CentralDogmaExtension dogma = new CentralDogmaExtension() {
+ @Override
+ protected void configure(CentralDogmaBuilder builder) {
+ builder.mirroringEnabled(true);
+ builder.maxNumBytesPerMirror(MAX_NUM_BYTES);
+ }
+ };
+
+ @RegisterExtension
+ static final ServerExtension server = new ServerExtension() {
+ @Override
+ protected void configure(ServerBuilder sb) throws Exception {
+ sb.annotatedService(new Object() {
+ @Get("/get/:length")
+ public String get(@Param int length) {
+ return Strings.repeat(".", length);
+ }
+
+ @Get("/204") // Generate a '204 No Content' response.
+ public void respond204() {}
+
+ @Get("/304")
+ @StatusCode(304)
+ public void respond304() {}
+ });
+ }
+ };
+
+ private static CentralDogma client;
+ private static MirroringService mirroringService;
+
+ @BeforeAll
+ static void init() {
+ client = dogma.client();
+ mirroringService = dogma.mirroringService();
+ }
+
+ private String projName;
+
+ @BeforeEach
+ void initDogmaRepo(TestInfo testInfo) {
+ projName = TestUtil.normalizedDisplayName(testInfo);
+ client.createProject(projName).join();
+ client.createRepository(projName, REPO_FOO).join();
+ }
+
+ @AfterEach
+ void destroyDogmaRepo() {
+ client.removeProject(projName).join();
+ }
+
+ @Test
+ void simple() throws Exception {
+ // Configure the server to mirror http://.../get/7 into /bar.txt.
+ pushMirrorSettings(REPO_FOO, "/bar.txt", 7);
+ testSuccessfulMirror(".......\n");
+ }
+
+ @Test
+ void tooLargeContent() throws Exception {
+ // Configure the server to mirror http://.../get/<MAX_NUM_BYTES + 1> into /bar.txt.
+ pushMirrorSettings(REPO_FOO, "/bar.txt", MAX_NUM_BYTES + 1);
+ testFailedMirror(ContentTooLargeException.class);
+ }
+
+ @Test
+ void shouldHandle204() throws Exception {
+ // Configure the server to mirror http://.../204 into /bar.txt.
+ pushMirrorSettings(REPO_FOO, "/bar.txt", server.httpUri() + "/204");
+ testSuccessfulMirror("");
+ }
+
+ @Test
+ void shouldRejectNon2xx() throws Exception {
+ // Configure the server to mirror http://.../get/<MAX_NUM_BYTES + 1> into /bar.txt.
+ pushMirrorSettings(REPO_FOO, "/bar.txt", server.httpUri() + "/304");
+ testFailedMirror(HttpStatusException.class);
+ }
+
+ private void pushMirrorSettings(String localRepo, @Nullable String localPath, int length) {
+ pushMirrorSettings(localRepo, localPath, remoteUri(length));
+ }
+
+ private void pushMirrorSettings(String localRepo, @Nullable String localPath, String remoteUri) {
+ client.forRepo(projName, Project.REPO_META)
+ .commit("Add /mirrors.json",
+ Change.ofJsonUpsert("/mirrors.json",
+ "[{" +
+ " \"type\": \"single\"," +
+ " \"direction\": \"REMOTE_TO_LOCAL\"," +
+ " \"localRepo\": \"" + localRepo + "\"," +
+ " \"localPath\": \"" + firstNonNull(localPath, "/") + "\"," +
+ " \"remoteUri\": \"" + remoteUri + "\"," +
+ " \"schedule\": \"0 0 0 1 1 ? 2099\"" +
+ "}]"))
+ .push().join();
+ }
+
+ private void testSuccessfulMirror(String expectedContent) {
+ // Trigger the mirroring task.
+ mirroringService.mirror().join();
+
+ // On successful mirroring, /bar.txt should contain 7 periods.
+ final Entry<?> entry = client.forRepo(projName, REPO_FOO)
+ .file("/bar.txt")
+ .get().join();
+
+ assertThat(entry.contentAsText()).isEqualTo(expectedContent);
+ }
+
+ private void testFailedMirror(Class<? extends Throwable> rootCause) {
+ // Trigger the mirroring task, which will fail because the response was too large.
+ assertThatThrownBy(() -> mirroringService.mirror().join()).cause().satisfies(cause -> {
+ assertThat(cause).isInstanceOf(MirrorException.class)
+ .hasCauseInstanceOf(rootCause);
+ });
+
+ // As a result, /bar.txt shouldn't exist.
+ assertThatThrownBy(() -> {
+ client.forRepo(projName, REPO_FOO)
+ .file("/bar.txt")
+ .get().join();
+ }).hasCauseInstanceOf(EntryNotFoundException.class);
+ }
+
+ private static String remoteUri(int length) {
+ return String.format("%s/get/%d", server.httpUri(), length);
+ }
+}

server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/AbstractMirror.java

@@ -52,7 +52,8 @@ public abstract class AbstractMirror implements Mirror {
  private final Repository localRepo;
  private final String localPath;
  private final URI remoteRepoUri;
- private final String remotePath;
+ @Nullable
+ private final String remoteSubpath;
  @Nullable
  private final String remoteBranch;
  @Nullable
@@ -62,16 +63,16 @@ public abstract class AbstractMirror implements Mirror {
 
  protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential,
  Repository localRepo, String localPath,
- URI remoteRepoUri, String remotePath, @Nullable String remoteBranch,
+ URI remoteRepoUri, @Nullable String remoteSubpath, @Nullable String remoteBranch,
  @Nullable String gitignore) {
 
  this.schedule = requireNonNull(schedule, "schedule");
  this.direction = requireNonNull(direction, "direction");
  this.credential = requireNonNull(credential, "credential");
  this.localRepo = requireNonNull(localRepo, "localRepo");
- this.localPath = normalizePath(requireNonNull(localPath, "localPath"));
+ this.localPath = requireNonNull(localPath, "localPath");
  this.remoteRepoUri = requireNonNull(remoteRepoUri, "remoteRepoUri");
- this.remotePath = normalizePath(requireNonNull(remotePath, "remotePath"));
+ this.remoteSubpath = remoteSubpath;
  this.remoteBranch = remoteBranch;
  this.gitignore = gitignore;
 
@@ -81,7 +82,7 @@ protected AbstractMirror(Cron schedule, MirrorDirection direction, MirrorCredent
  // Use the properties' hash code so that the same properties result in the same jitter.
  jitterMillis = Math.abs(Objects.hash(this.schedule.asString(), this.direction,
  this.localRepo.parent().name(), this.localRepo.name(),
- this.remoteRepoUri, this.remotePath, this.remoteBranch) /
+ this.remoteRepoUri, this.remoteSubpath, this.remoteBranch) /
  (Integer.MAX_VALUE / 60000));
  }
 
@@ -129,8 +130,8 @@ public final URI remoteRepoUri() {
  }
 
  @Override
- public final String remotePath() {
- return remotePath;
+ public String remoteSubpath() {
+ return remoteSubpath;
  }
 
  @Override
@@ -180,7 +181,7 @@ public String toString() {
  .add("localRepo", localRepo.name())
  .add("localPath", localPath)
  .add("remoteRepo", remoteRepoUri)
- .add("remotePath", remotePath)
+ .add("remoteSubpath", remoteSubpath)
  .add("remoteBranch", remoteBranch)
  .add("gitignore", gitignore)
  .add("credential", credential);

server/src/main/java/com/linecorp/centraldogma/server/internal/mirror/CentralDogmaMirror.java

@@ -16,11 +16,13 @@
 
 package com.linecorp.centraldogma.server.internal.mirror;
 
+import static com.linecorp.centraldogma.server.mirror.MirrorUtil.normalizePath;
 import static java.util.Objects.requireNonNull;
 
 import java.io.File;
 import java.net.URI;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import com.cronutils.model.Cron;
@@ -37,16 +39,31 @@ public final class CentralDogmaMirror extends AbstractMirror {
 
  public CentralDogmaMirror(Cron schedule, MirrorDirection direction, MirrorCredential credential,
  Repository localRepo, String localPath,
- URI remoteRepoUri, String remoteProject, String remoteRepo, String remotePath,
+ URI remoteRepoUri, String remoteProject, String remoteRepo, String remoteSubpath,
  @Nullable String gitignore) {
  // Central Dogma has no notion of 'branch', so we just pass null as a placeholder.
- super(schedule, direction, credential, localRepo, localPath, remoteRepoUri, remotePath, null,
+ super(schedule,
+ direction,
+ credential,
+ localRepo,
+ normalizePath(requireNonNull(localPath, "localPath")),
+ remoteRepoUri,
+ normalizePath(requireNonNull(remoteSubpath, "remoteSubpath")),
+ null,
  gitignore);
 
  this.remoteProject = requireNonNull(remoteProject, "remoteProject");
  this.remoteRepo = requireNonNull(remoteRepo, "remoteRepo");
  }
 
+ @Nonnull
+ @Override
+ public String remoteSubpath() {
+ final String remoteSubpath = super.remoteSubpath();
+ assert remoteSubpath != null;
+ return remoteSubpath;
+ }
+
  String remoteProject() {
  return remoteProject;
  }