From 9b1c73ed23160ca671a679f9b42b09f880c74b85 Mon Sep 17 00:00:00 2001 From: Luck Date: Mon, 7 Jan 2019 16:56:16 +0000 Subject: [PATCH] Implement use of a Maven Central mirror for dependency downloads People keep telling me that LP's use of Maven Central for downloading dependencies is not allowed / inappropriate / abusive. I disagree but I'm bored of hearing it. Using a mirror will mean that all of the load is taken off of Central, and is instead absorbed by my servers + (mostly) Cloudflare. - The mirror is (currently) hosted at https://nexus.lucko.me/repository/maven-central/ - The prospect of the mirror becoming compromised is not a concern. LuckPerms compares the downloaded content against a checksum before saving it. - The prospect of the mirror going offline is also not a concern. We will fallback to Maven Central if a connection cannot be made to the mirror. --- .../common/dependencies/Dependency.java | 68 +++++++++------- .../dependencies/DependencyManager.java | 79 +++++++++++++------ 2 files changed, 97 insertions(+), 50 deletions(-) diff --git a/common/src/main/java/me/lucko/luckperms/common/dependencies/Dependency.java b/common/src/main/java/me/lucko/luckperms/common/dependencies/Dependency.java index 019389f4..6a35dd77 100644 --- a/common/src/main/java/me/lucko/luckperms/common/dependencies/Dependency.java +++ b/common/src/main/java/me/lucko/luckperms/common/dependencies/Dependency.java @@ -32,8 +32,11 @@ import me.lucko.luckperms.common.dependencies.relocation.Relocation; import me.lucko.luckperms.common.dependencies.relocation.RelocationHelper; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; +import java.net.URLConnection; import java.security.MessageDigest; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.List; @@ -265,12 +268,14 @@ public enum Dependency { Relocation.of("toml4j", "com{}moandjiezana{}toml") ); - private final String url; + private final List urls; private final String version; private final byte[] checksum; private final List relocations; - private static final String MAVEN_CENTRAL_FORMAT = "https://repo1.maven.org/maven2/%s/%s/%s/%s-%s.jar"; + private static final String MAVEN_CENTRAL_REPO = "https://repo1.maven.org/maven2/"; + private static final String LUCK_MIRROR_REPO = "https://nexus.lucko.me/repository/maven-central/"; + private static final String MAVEN_FORMAT = "%s/%s/%s/%s-%s.jar"; Dependency(String groupId, String artifactId, String version, String checksum) { this(groupId, artifactId, version, checksum, ImmutableList.of()); @@ -281,20 +286,21 @@ public enum Dependency { } Dependency(String groupId, String artifactId, String version, String checksum, List relocations) { - this( - String.format(MAVEN_CENTRAL_FORMAT, - rewriteEscaping(groupId).replace(".", "/"), - rewriteEscaping(artifactId), - version, - rewriteEscaping(artifactId), - version - ), - version, checksum, relocations + String path = String.format(MAVEN_FORMAT, + rewriteEscaping(groupId).replace(".", "/"), + rewriteEscaping(artifactId), + version, + rewriteEscaping(artifactId), + version ); - } - - Dependency(String url, String version, String checksum, List relocations) { - this.url = url; + try { + this.urls = ImmutableList.of( + new URL(LUCK_MIRROR_REPO + path), + new URL(MAVEN_CENTRAL_REPO + path) + ); + } catch (MalformedURLException e) { + throw new RuntimeException(e); // propagate + } this.version = version; this.checksum = Base64.getDecoder().decode(checksum); this.relocations = ImmutableList.copyOf(relocations); @@ -308,26 +314,32 @@ public enum Dependency { MessageDigest digest = MessageDigest.getInstance("SHA-256"); for (Dependency dependency : values()) { - URL url = new URL(dependency.getUrl()); - try (InputStream in = url.openStream()) { - byte[] bytes = ByteStreams.toByteArray(in); - if (bytes.length == 0) { - throw new RuntimeException("Empty stream"); + List hashes = new ArrayList<>(); + for (URL url : dependency.getUrls()) { + URLConnection connection = url.openConnection(); + connection.setRequestProperty("User-Agent", "luckperms"); + + try (InputStream in = connection.getInputStream()) { + byte[] bytes = ByteStreams.toByteArray(in); + if (bytes.length == 0) { + throw new RuntimeException("Empty stream"); + } + + hashes.add(digest.digest(bytes)); } + } - byte[] hash = digest.digest(bytes); - - if (Arrays.equals(hash, dependency.getChecksum())) { - System.out.println("MATCH " + dependency.name() + ": " + Base64.getEncoder().encodeToString(hash)); - } else { - System.out.println("NO MATCH " + dependency.name() + ": " + Base64.getEncoder().encodeToString(hash)); + for (int i = 0; i < hashes.size(); i++) { + byte[] hash = hashes.get(i); + if (!Arrays.equals(hash, dependency.getChecksum())) { + System.out.println("NO MATCH - REPO " + i + " - " + dependency.name() + ": " + Base64.getEncoder().encodeToString(hash)); } } } } - public String getUrl() { - return this.url; + public List getUrls() { + return this.urls; } public String getVersion() { diff --git a/common/src/main/java/me/lucko/luckperms/common/dependencies/DependencyManager.java b/common/src/main/java/me/lucko/luckperms/common/dependencies/DependencyManager.java index ed01a89f..c28935e7 100644 --- a/common/src/main/java/me/lucko/luckperms/common/dependencies/DependencyManager.java +++ b/common/src/main/java/me/lucko/luckperms/common/dependencies/DependencyManager.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; @@ -51,6 +52,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * Responsible for loading runtime dependencies. @@ -209,35 +211,68 @@ public class DependencyManager { return file; } - URL url = new URL(dependency.getUrl()); - try (InputStream in = url.openStream()) { + boolean success = false; + Exception lastError = null; - // download the jar content - byte[] bytes = ByteStreams.toByteArray(in); - if (bytes.length == 0) { - throw new RuntimeException("Empty stream"); + // getUrls returns two possible sources of the dependency. + // [0] is a mirror of Maven Central, used to reduce load on central. apparently they don't like being used as a CDN + // [1] is Maven Central itself + + // side note: the relative "security" of the mirror is less than central, but it actually doesn't matter. + // we compare the downloaded file against a checksum here, so even if the mirror became compromised, RCE wouldn't be possible. + // if the mirror download doesn't match the checksum, we just try maven central instead. + + List urls = dependency.getUrls(); + for (int i = 0; i < urls.size() && !success; i++) { + URL url = urls.get(i); + + try { + URLConnection connection = url.openConnection(); + + // i == 0 when we're trying to use the mirror repo. + // set some timeout properties so when/if this repository goes offline, we quickly fallback to central. + if (i == 0) { + connection.setRequestProperty("User-Agent", "luckperms"); + connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(5)); + connection.setReadTimeout((int) TimeUnit.SECONDS.toMillis(10)); + } + + try (InputStream in = connection.getInputStream()) { + // download the jar content + byte[] bytes = ByteStreams.toByteArray(in); + if (bytes.length == 0) { + throw new RuntimeException("Empty stream"); + } + + + // compute a hash for the downloaded file + byte[] hash = this.digest.digest(bytes); + + // ensure the hash matches the expected checksum + if (!Arrays.equals(hash, dependency.getChecksum())) { + throw new RuntimeException("Downloaded file had an invalid hash. " + + "Expected: " + Base64.getEncoder().encodeToString(dependency.getChecksum()) + " " + + "Actual: " + Base64.getEncoder().encodeToString(hash)); + } + + this.plugin.getLogger().info("Successfully downloaded '" + fileName + "' with matching checksum: " + Base64.getEncoder().encodeToString(hash)); + + // if the checksum matches, save the content to disk + Files.write(file, bytes); + success = true; + } + } catch (Exception e) { + lastError = e; } + } - - // compute a hash for the downloaded file - byte[] hash = this.digest.digest(bytes); - - // ensure the hash matches the expected checksum - if (!Arrays.equals(hash, dependency.getChecksum())) { - throw new RuntimeException("Downloaded file had an invalid hash. " + - "Expected: " + Base64.getEncoder().encodeToString(dependency.getChecksum()) + " " + - "Actual: " + Base64.getEncoder().encodeToString(hash)); - } - - this.plugin.getLogger().info("Successfully downloaded '" + fileName + "' with matching checksum: " + Base64.getEncoder().encodeToString(hash)); - - // if the checksum matches, save the content to disk - Files.write(file, bytes); + if (!success) { + throw new RuntimeException("Unable to download", lastError); } // ensure the file saved correctly if (!Files.exists(file)) { - throw new IllegalStateException("File not present. - " + file.toString()); + throw new IllegalStateException("File not present: " + file.toString()); } else { return file; }