diff --git a/bukkit/src/main/java/me/lucko/luckperms/bukkit/LPBukkitPlugin.java b/bukkit/src/main/java/me/lucko/luckperms/bukkit/LPBukkitPlugin.java index 831b8465..b74decb4 100644 --- a/bukkit/src/main/java/me/lucko/luckperms/bukkit/LPBukkitPlugin.java +++ b/bukkit/src/main/java/me/lucko/luckperms/bukkit/LPBukkitPlugin.java @@ -49,7 +49,6 @@ import me.lucko.luckperms.common.contexts.ContextManager; import me.lucko.luckperms.common.contexts.ServerCalculator; import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.model.User; -import me.lucko.luckperms.common.data.Importer; import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.dependencies.DependencyManager; import me.lucko.luckperms.common.locale.LocaleManager; @@ -112,7 +111,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin { private BukkitListener listener; private ApiProvider apiProvider; private Logger log; - private Importer importer; private DefaultsProvider defaultsProvider; private ChildPermissionProvider childPermissionProvider; private LocaleManager localeManager; @@ -247,7 +245,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin { userManager = new GenericUserManager(this); groupManager = new GenericGroupManager(this); trackManager = new GenericTrackManager(this); - importer = new Importer(commandManager); calculatorFactory = new BukkitCalculatorFactory(this); cachedStateManager = new CachedStateManager(this); @@ -371,7 +368,6 @@ public class LPBukkitPlugin extends JavaPlugin implements LuckPermsPlugin { listener = null; apiProvider = null; log = null; - importer = null; defaultsProvider = null; childPermissionProvider = null; localeManager = null; diff --git a/bukkit/src/main/java/me/lucko/luckperms/bukkit/migration/MigrationPowerfulPerms.java b/bukkit/src/main/java/me/lucko/luckperms/bukkit/migration/MigrationPowerfulPerms.java index 797b5db8..8646c40d 100644 --- a/bukkit/src/main/java/me/lucko/luckperms/bukkit/migration/MigrationPowerfulPerms.java +++ b/bukkit/src/main/java/me/lucko/luckperms/bukkit/migration/MigrationPowerfulPerms.java @@ -111,7 +111,7 @@ public class MigrationPowerfulPerms extends SubCommand { Set uuids = new HashSet<>(); try (HikariSupplier hikari = new HikariSupplier(address, database, username, password)) { - hikari.setup(); + hikari.setup("powerfulperms-migrator-pool"); try (Connection c = hikari.getConnection()) { DatabaseMetaData meta = c.getMetaData(); diff --git a/bungee/src/main/java/me/lucko/luckperms/bungee/LPBungeePlugin.java b/bungee/src/main/java/me/lucko/luckperms/bungee/LPBungeePlugin.java index 6c79fbdf..f64d22c8 100644 --- a/bungee/src/main/java/me/lucko/luckperms/bungee/LPBungeePlugin.java +++ b/bungee/src/main/java/me/lucko/luckperms/bungee/LPBungeePlugin.java @@ -42,7 +42,6 @@ import me.lucko.luckperms.common.contexts.ContextManager; import me.lucko.luckperms.common.contexts.ServerCalculator; import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.model.User; -import me.lucko.luckperms.common.data.Importer; import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.dependencies.DependencyManager; import me.lucko.luckperms.common.locale.LocaleManager; @@ -85,6 +84,7 @@ import java.util.stream.Collectors; public class LPBungeePlugin extends Plugin implements LuckPermsPlugin { private final Set ignoringLogs = ConcurrentHashMap.newKeySet(); private LuckPermsScheduler scheduler; + private CommandManager commandManager; private LuckPermsConfiguration configuration; private UserManager userManager; private GroupManager groupManager; @@ -94,7 +94,6 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin { private UuidCache uuidCache; private ApiProvider apiProvider; private Logger log; - private Importer importer; private LocaleManager localeManager; private CachedStateManager cachedStateManager; private ContextManager contextManager; @@ -181,7 +180,7 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin { // register commands getLog().info("Registering commands..."); - CommandManager commandManager = new CommandManager(this); + commandManager = new CommandManager(this); getProxy().getPluginManager().registerCommand(this, new BungeeCommand(this, commandManager)); // disable the default Bungee /perms command so it gets handled by the Bukkit plugin @@ -193,7 +192,6 @@ public class LPBungeePlugin extends Plugin implements LuckPermsPlugin { userManager = new GenericUserManager(this); groupManager = new GenericGroupManager(this); trackManager = new GenericTrackManager(this); - importer = new Importer(commandManager); calculatorFactory = new BungeeCalculatorFactory(this); cachedStateManager = new CachedStateManager(this); diff --git a/common/src/main/java/me/lucko/luckperms/common/commands/misc/ExportCommand.java b/common/src/main/java/me/lucko/luckperms/common/commands/misc/ExportCommand.java index 3414ba25..20bcb4ba 100644 --- a/common/src/main/java/me/lucko/luckperms/common/commands/misc/ExportCommand.java +++ b/common/src/main/java/me/lucko/luckperms/common/commands/misc/ExportCommand.java @@ -22,47 +22,25 @@ package me.lucko.luckperms.common.commands.misc; -import me.lucko.luckperms.api.Node; import me.lucko.luckperms.common.commands.Arg; import me.lucko.luckperms.common.commands.CommandResult; import me.lucko.luckperms.common.commands.SingleCommand; import me.lucko.luckperms.common.commands.sender.Sender; import me.lucko.luckperms.common.constants.Message; import me.lucko.luckperms.common.constants.Permission; -import me.lucko.luckperms.common.core.NodeFactory; -import me.lucko.luckperms.common.core.model.Group; -import me.lucko.luckperms.common.core.model.Track; -import me.lucko.luckperms.common.core.model.User; +import me.lucko.luckperms.common.data.Exporter; import me.lucko.luckperms.common.plugin.LuckPermsPlugin; -import me.lucko.luckperms.common.storage.Storage; import me.lucko.luckperms.common.utils.Predicates; -import me.lucko.luckperms.common.utils.ProgressLogger; -import java.io.BufferedWriter; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Date; import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; public class ExportCommand extends SingleCommand { - private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); - - private static void write(BufferedWriter writer, String s) { - try { - writer.write(s); - writer.newLine(); - } catch (IOException e) { - e.printStackTrace(); - } - } + private AtomicBoolean running = new AtomicBoolean(false); public ExportCommand() { super("Export", "Export data to a file", "/%s export ", Permission.EXPORT, Predicates.not(1), @@ -74,9 +52,10 @@ public class ExportCommand extends SingleCommand { @Override public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List args, String label) { - ProgressLogger log = new ProgressLogger(null, Message.EXPORT_LOG, Message.EXPORT_LOG_PROGRESS); - log.addListener(plugin.getConsoleSender()); - log.addListener(sender); + if (running.get()) { + Message.EXPORT_ALREADY_RUNNING.send(sender); + return CommandResult.STATE_ERROR; + } File f = new File(plugin.getDataDirectory(), args.get(0)); if (f.exists()) { @@ -99,112 +78,23 @@ public class ExportCommand extends SingleCommand { return CommandResult.FAILURE; } - try (BufferedWriter writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8)) { - log.log("Starting."); - - write(writer, "# LuckPerms Export File"); - write(writer, "# Generated by " + sender.getName() + " at " + DATE_FORMAT.format(new Date(System.currentTimeMillis()))); - write(writer, ""); - - // Export Groups - log.log("Starting group export."); - - // Create the actual groups first - write(writer, "# Create groups"); - for (Group group : plugin.getGroupManager().getAll().values()) { - write(writer, "/luckperms creategroup " + group.getName()); - } - write(writer, ""); - - AtomicInteger groupCount = new AtomicInteger(0); - for (Group group : plugin.getGroupManager().getAll().values()) { - write(writer, "# Export group: " + group.getName()); - for (Node node : group.getNodes()) { - write(writer, NodeFactory.nodeAsCommand(node, group.getName(), true)); - } - write(writer, ""); - log.logAllProgress("Exported {} groups so far.", groupCount.incrementAndGet()); - } - log.log("Exported " + groupCount.get() + " groups."); - - write(writer, ""); - write(writer, ""); - - // Export tracks - log.log("Starting track export."); - - Collection tracks = plugin.getTrackManager().getAll().values(); - if (!tracks.isEmpty()) { - - // Create the actual tracks first - write(writer, "# Create tracks"); - for (Track track : tracks) { - write(writer, "/luckperms createtrack " + track.getName()); - } - - write(writer, ""); - - AtomicInteger trackCount = new AtomicInteger(0); - for (Track track : plugin.getTrackManager().getAll().values()) { - write(writer, "# Export track: " + track.getName()); - for (String group : track.getGroups()) { - write(writer, "/luckperms track " + track.getName() + " append " + group); - } - write(writer, ""); - log.logAllProgress("Exported {} tracks so far.", trackCount.incrementAndGet()); - } - - write(writer, ""); - write(writer, ""); - } - - log.log("Exported " + tracks.size() + " tracks."); - - // Export users - log.log("Starting user export. Finding a list of unique users to export."); - Storage ds = plugin.getStorage(); - Set users = ds.getUniqueUsers().join(); - log.log("Found " + users.size() + " unique users to export."); - - write(writer, "# Export users"); - - AtomicInteger userCount = new AtomicInteger(0); - for (UUID uuid : users) { - plugin.getStorage().loadUser(uuid, "null").join(); - User user = plugin.getUserManager().get(uuid); - write(writer, "# Export user: " + user.getUuid().toString() + " - " + user.getName()); - - boolean inDefault = false; - for (Node node : user.getNodes()) { - if (node.isGroupNode() && node.getGroupName().equalsIgnoreCase("default")) { - inDefault = true; - continue; - } - - write(writer, NodeFactory.nodeAsCommand(node, user.getUuid().toString(), false)); - } - - if (!user.getPrimaryGroup().equalsIgnoreCase("default")) { - write(writer, "/luckperms user " + user.getUuid().toString() + " switchprimarygroup " + user.getPrimaryGroup()); - } - - if (!inDefault) { - write(writer, "/luckperms user " + user.getUuid().toString() + " parent remove default"); - } - - plugin.getUserManager().cleanup(user); - log.logProgress("Exported {} users so far.", userCount.incrementAndGet()); - } - log.log("Exported " + userCount.get() + " users."); - - writer.flush(); - - log.getListeners().forEach(l -> Message.LOG_EXPORT_SUCCESS.send(l, f.getAbsolutePath())); - return CommandResult.SUCCESS; - } catch (Throwable t) { - t.printStackTrace(); - return CommandResult.FAILURE; + if (!running.compareAndSet(false, true)) { + Message.EXPORT_ALREADY_RUNNING.send(sender); + return CommandResult.STATE_ERROR; } + + Exporter exporter = new Exporter(plugin, sender, path); + + // Run the exporter in its own thread. + plugin.doAsync(() -> { + try { + exporter.run(); + } finally { + running.set(false); + } + }); + + return CommandResult.SUCCESS; } } diff --git a/common/src/main/java/me/lucko/luckperms/common/commands/misc/ImportCommand.java b/common/src/main/java/me/lucko/luckperms/common/commands/misc/ImportCommand.java index d81d7b81..d74b6aa6 100644 --- a/common/src/main/java/me/lucko/luckperms/common/commands/misc/ImportCommand.java +++ b/common/src/main/java/me/lucko/luckperms/common/commands/misc/ImportCommand.java @@ -38,8 +38,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; public class ImportCommand extends SingleCommand { + private AtomicBoolean running = new AtomicBoolean(false); + public ImportCommand() { super("Import", "Import data from a file", "/%s import ", Permission.IMPORT, Predicates.not(1), Arg.list( @@ -50,13 +53,11 @@ public class ImportCommand extends SingleCommand { @Override public CommandResult execute(LuckPermsPlugin plugin, Sender sender, List args, String label) { - if (args.size() == 0) { - sendUsage(sender, label); - return CommandResult.INVALID_ARGS; + if (running.get()) { + Message.IMPORT_ALREADY_RUNNING.send(sender); + return CommandResult.STATE_ERROR; } - Importer importer = plugin.getImporter(); - File f = new File(plugin.getDataDirectory(), args.get(0)); if (!f.exists()) { Message.IMPORT_LOG_DOESNT_EXIST.send(sender, f.getAbsolutePath()); @@ -80,13 +81,22 @@ public class ImportCommand extends SingleCommand { return CommandResult.FAILURE; } - if (!importer.startRun()) { + if (!running.compareAndSet(false, true)) { Message.IMPORT_ALREADY_RUNNING.send(sender); return CommandResult.STATE_ERROR; } + Importer importer = new Importer(plugin.getCommandManager(), sender, commands); + // Run the importer in its own thread. - plugin.doAsync(() -> importer.start(sender, commands)); + plugin.doAsync(() -> { + try { + importer.run(); + } finally { + running.set(false); + } + }); + return CommandResult.SUCCESS; } } diff --git a/common/src/main/java/me/lucko/luckperms/common/commands/utils/Util.java b/common/src/main/java/me/lucko/luckperms/common/commands/utils/Util.java index 30837738..4b6f054a 100644 --- a/common/src/main/java/me/lucko/luckperms/common/commands/utils/Util.java +++ b/common/src/main/java/me/lucko/luckperms/common/commands/utils/Util.java @@ -341,7 +341,7 @@ public class Util { return Maps.immutableEntry(message, title); } - public static List> divideList(List source, int size) { + public static List> divideList(Iterable source, int size) { List> lists = new ArrayList<>(); Iterator it = source.iterator(); while (it.hasNext()) { diff --git a/common/src/main/java/me/lucko/luckperms/common/constants/Message.java b/common/src/main/java/me/lucko/luckperms/common/constants/Message.java index b54c6e9a..a5cb01c1 100644 --- a/common/src/main/java/me/lucko/luckperms/common/constants/Message.java +++ b/common/src/main/java/me/lucko/luckperms/common/constants/Message.java @@ -410,6 +410,7 @@ public enum Message { LOG_EXPORT_SUCCESS("&aSuccessfully exported the log to &b{0}&a.", true), IMPORT_ALREADY_RUNNING("Another import process is already running. Please wait for it to finish and try again.", true), + EXPORT_ALREADY_RUNNING("Another export process is already running. Please wait for it to finish and try again.", true), IMPORT_LOG_DOESNT_EXIST("Error: File {0} does not exist.", true), IMPORT_LOG_NOT_READABLE("Error: File {0} is not readable.", true), IMPORT_LOG_FAILURE("An unexpected error occured whilst reading from the log file.", true), diff --git a/common/src/main/java/me/lucko/luckperms/common/core/model/User.java b/common/src/main/java/me/lucko/luckperms/common/core/model/User.java index ab06e28c..6cbda6e0 100644 --- a/common/src/main/java/me/lucko/luckperms/common/core/model/User.java +++ b/common/src/main/java/me/lucko/luckperms/common/core/model/User.java @@ -143,7 +143,7 @@ public class User extends PermissionHolder implements Identifiable + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package me.lucko.luckperms.common.data; + +import me.lucko.luckperms.api.Node; +import me.lucko.luckperms.common.commands.sender.Sender; +import me.lucko.luckperms.common.commands.utils.Util; +import me.lucko.luckperms.common.constants.Message; +import me.lucko.luckperms.common.core.NodeFactory; +import me.lucko.luckperms.common.core.model.Group; +import me.lucko.luckperms.common.core.model.Track; +import me.lucko.luckperms.common.core.model.User; +import me.lucko.luckperms.common.plugin.LuckPermsPlugin; +import me.lucko.luckperms.common.storage.Storage; +import me.lucko.luckperms.common.utils.ProgressLogger; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * Handles export operations + */ +public class Exporter implements Runnable { + private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); + + // number of users --> the value to divide the list by. base value of 1000 per thread, with a max of 10 threads. so, 3000 = 3 threads, 10000+ = 10 threads + private static final Function THREAD_COUNT_FUNCTION = usersCount -> { + // how many threads to make. must be 1 <= x <= 15 + int i = Math.max(1, Math.min(15, usersCount / 1000)); + + // then work out the value to split at. e.g. if we have 1,000 users to export and 2 threads, we should split at every 500 users. + return Math.max(1, usersCount / i); + }; + + private static void write(BufferedWriter writer, String s) { + try { + writer.write(s); + writer.newLine(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private final LuckPermsPlugin plugin; + private final Sender executor; + private final Path filePath; + private final ProgressLogger log; + + public Exporter(LuckPermsPlugin plugin, Sender executor, Path filePath) { + this.plugin = plugin; + this.executor = executor; + this.filePath = filePath; + + log = new ProgressLogger(null, Message.EXPORT_LOG, Message.EXPORT_LOG_PROGRESS); + log.addListener(plugin.getConsoleSender()); + log.addListener(executor); + } + + @Override + public void run() { + try (BufferedWriter writer = Files.newBufferedWriter(filePath, StandardCharsets.UTF_8)) { + log.log("Starting."); + + write(writer, "# LuckPerms Export File"); + write(writer, "# Generated by " + executor.getName() + " at " + DATE_FORMAT.format(new Date(System.currentTimeMillis()))); + write(writer, ""); + + // Export Groups + log.log("Starting group export."); + + // Create the actual groups first + write(writer, "# Create groups"); + for (Group group : plugin.getGroupManager().getAll().values()) { + write(writer, "/luckperms creategroup " + group.getName()); + } + write(writer, ""); + + AtomicInteger groupCount = new AtomicInteger(0); + for (Group group : plugin.getGroupManager().getAll().values()) { + write(writer, "# Export group: " + group.getName()); + for (Node node : group.getNodes()) { + write(writer, NodeFactory.nodeAsCommand(node, group.getName(), true)); + } + write(writer, ""); + log.logAllProgress("Exported {} groups so far.", groupCount.incrementAndGet()); + } + log.log("Exported " + groupCount.get() + " groups."); + + write(writer, ""); + write(writer, ""); + + // Export tracks + log.log("Starting track export."); + + Collection tracks = plugin.getTrackManager().getAll().values(); + if (!tracks.isEmpty()) { + + // Create the actual tracks first + write(writer, "# Create tracks"); + for (Track track : tracks) { + write(writer, "/luckperms createtrack " + track.getName()); + } + + write(writer, ""); + + AtomicInteger trackCount = new AtomicInteger(0); + for (Track track : plugin.getTrackManager().getAll().values()) { + write(writer, "# Export track: " + track.getName()); + for (String group : track.getGroups()) { + write(writer, "/luckperms track " + track.getName() + " append " + group); + } + write(writer, ""); + log.logAllProgress("Exported {} tracks so far.", trackCount.incrementAndGet()); + } + + write(writer, ""); + write(writer, ""); + } + + log.log("Exported " + tracks.size() + " tracks."); + + + // Users are migrated in separate threads. + // This is because there are likely to be a lot of them, and because we can. + // It's a big speed improvement, since the database/files are split up and can handle concurrent reads. + + log.log("Starting user export. Finding a list of unique users to export."); + + // Find all of the unique users we need to export + Storage ds = plugin.getStorage(); + Set users = ds.getUniqueUsers().join(); + log.log("Found " + users.size() + " unique users to export."); + + write(writer, "# Export users"); + + List> subUsers; + AtomicInteger userCount = new AtomicInteger(0); + + // not really that many users, so it's not really worth spreading the load. + if (users.size() < 1500) { + subUsers = Collections.singletonList(new ArrayList<>(users)); + } else { + subUsers = Util.divideList(users, THREAD_COUNT_FUNCTION.apply(users.size())); + } + + log.log("Split users into " + subUsers.size() + " threads for export."); + + // Setup a file writing lock. We don't want multiple threads writing at the same time. + // The write function accepts a list of strings, as we want a user's data to be grouped together. + // This means it can be processed and added in one go. + ReentrantLock lock = new ReentrantLock(); + Consumer> writeFunction = strings -> { + lock.lock(); + try { + for (String s : strings) { + write(writer, s); + } + } finally { + lock.unlock(); + } + }; + + // A set of futures, which are really just the threads we need to wait for. + Set> futures = new HashSet<>(); + + // iterate through each user sublist. + for (List subList : subUsers) { + + // register and start a new thread to process the sublist + futures.add(CompletableFuture.runAsync(() -> { + + // iterate through each user in the sublist, and grab their data. + for (UUID uuid : subList) { + try { + // actually export the user. this output will be fed to the writing function when we have all of the user's data. + List output = new ArrayList<>(); + + plugin.getStorage().loadUser(uuid, "null").join(); + User user = plugin.getUserManager().get(uuid); + output.add("# Export user: " + user.getUuid().toString() + " - " + user.getName()); + + boolean inDefault = false; + for (Node node : user.getNodes()) { + if (node.isGroupNode() && node.getGroupName().equalsIgnoreCase("default")) { + inDefault = true; + continue; + } + + output.add(NodeFactory.nodeAsCommand(node, user.getUuid().toString(), false)); + } + + if (!user.getPrimaryGroup().equalsIgnoreCase("default")) { + output.add("/luckperms user " + user.getUuid().toString() + " switchprimarygroup " + user.getPrimaryGroup()); + } + + if (!inDefault) { + output.add("/luckperms user " + user.getUuid().toString() + " parent remove default"); + } + + plugin.getUserManager().cleanup(user); + writeFunction.accept(output); + + log.logProgress("Exported {} users so far.", userCount.incrementAndGet()); + + } catch (Exception e) { + e.printStackTrace(); + } + } + }, plugin.getScheduler().getAsyncExecutor())); + } + + // all of the threads have been scheduled now and are running. we just need to wait for them all to complete + CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).join(); + + log.log("Exported " + userCount.get() + " users."); + + writer.flush(); + log.getListeners().forEach(l -> Message.LOG_EXPORT_SUCCESS.send(l, filePath.toFile().getAbsolutePath())); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/common/src/main/java/me/lucko/luckperms/common/data/Importer.java b/common/src/main/java/me/lucko/luckperms/common/data/Importer.java index 70641651..6731e319 100644 --- a/common/src/main/java/me/lucko/luckperms/common/data/Importer.java +++ b/common/src/main/java/me/lucko/luckperms/common/data/Importer.java @@ -24,7 +24,6 @@ package me.lucko.luckperms.common.data; import lombok.AccessLevel; import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.Setter; import com.google.common.base.Splitter; @@ -48,33 +47,27 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; /** - * Class to handle import operations + * Handles import operations */ -@RequiredArgsConstructor -public class Importer { - private final CommandManager commandManager; +public class Importer implements Runnable { + private static final int PROGRESS_REPORT_SECONDS = 10; - private boolean running = false; - private Set notify = ImmutableSet.of(); - private List commands = null; - private Map cmdResult = null; + private final CommandManager commandManager; + private final Set notify; + private final List commands; + private final Map cmdResult; + private final FakeSender fake; private long lastMsg = 0; private int executing = -1; - public synchronized boolean startRun() { - if (running) { - return false; - } + public Importer(CommandManager commandManager, Sender executor, List commands) { + this.commandManager = commandManager; - running = true; - return true; - } - - public void start(Sender executor, List commands) { if (executor.isConsole()) { this.notify = ImmutableSet.of(executor); } else { @@ -90,29 +83,20 @@ public class Importer { .map(s -> s.startsWith("luckperms ") ? s.substring("luckperms ".length()) : s) .collect(Collectors.toList()); - cmdResult = new HashMap<>(); - - run(); - } - - private void cleanup() { - notify = ImmutableSet.of(); - commands = null; - cmdResult = null; - lastMsg = 0; - executing = -1; - running = false; + this.cmdResult = new HashMap<>(); + this.fake = new FakeSender(); } + @Override public void run() { long startTime = System.currentTimeMillis(); notify.forEach(s -> Message.IMPORT_START.send(s)); - final Sender fake = new FakeSender(this); int index = 1; for (String command : commands) { - if (lastMsg < (System.currentTimeMillis() / 1000) - 5) { - lastMsg = System.currentTimeMillis() / 1000; + long time = System.currentTimeMillis() / 1000; + if (lastMsg < (time - PROGRESS_REPORT_SECONDS)) { + lastMsg = time; sendProgress(index); } @@ -134,80 +118,57 @@ public class Importer { } long endTime = System.currentTimeMillis(); - double seconds = (endTime - startTime) / 1000.0; + double seconds = (endTime - startTime) / 1000; - int errors = 0; - for (Map.Entry e : cmdResult.entrySet()) { - if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) { - errors++; - } - } + int errors = (int) cmdResult.values().stream().filter(v -> v.getResult() != null && !v.getResult().asBoolean()).count(); if (errors == 0) { - for (Sender s : notify) { - Message.IMPORT_END_COMPLETE.send(s, seconds); - } + notify.forEach(s -> Message.IMPORT_END_COMPLETE.send(s, seconds)); } else if (errors == 1) { - for (Sender s : notify) { - Message.IMPORT_END_COMPLETE_ERR_SIN.send(s, seconds, errors); - } + notify.forEach(s -> Message.IMPORT_END_COMPLETE_ERR_SIN.send(s, seconds, errors)); } else { - for (Sender s : notify) { - Message.IMPORT_END_COMPLETE_ERR.send(s, seconds, errors); - } + notify.forEach(s -> Message.IMPORT_END_COMPLETE_ERR.send(s, seconds, errors)); } - int errIndex = 1; + AtomicInteger errIndex = new AtomicInteger(1); for (Map.Entry e : cmdResult.entrySet()) { if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) { - for (Sender s : notify) { - Message.IMPORT_END_ERROR_HEADER.send(s, errIndex, e.getKey(), e.getValue().getCommand(), e.getValue().getResult().toString()); + notify.forEach(s -> { + Message.IMPORT_END_ERROR_HEADER.send(s, errIndex.get(), e.getKey(), e.getValue().getCommand(), e.getValue().getResult().toString()); for (String out : e.getValue().getOutput()) { Message.IMPORT_END_ERROR_CONTENT.send(s, out); } Message.IMPORT_END_ERROR_FOOTER.send(s); - } + }); - errIndex++; + errIndex.incrementAndGet(); } } - - cleanup(); } private void sendProgress(int executing) { int percent = (executing * 100) / commands.size(); - int errors = 0; - - for (Map.Entry e : cmdResult.entrySet()) { - if (e.getValue().getResult() != null && !e.getValue().getResult().asBoolean()) { - errors++; - } - } + int errors = (int) cmdResult.values().stream().filter(v -> v.getResult() != null && !v.getResult().asBoolean()).count(); if (errors == 1) { - for (Sender s : notify) { - Message.IMPORT_PROGRESS_SIN.send(s, percent, executing, commands.size(), errors); - } + notify.forEach(s -> Message.IMPORT_PROGRESS_SIN.send(s, percent, executing, commands.size(), errors)); } else { - for (Sender s : notify) { - Message.IMPORT_PROGRESS.send(s, percent, executing, commands.size(), errors); - } + notify.forEach(s -> Message.IMPORT_PROGRESS.send(s, percent, executing, commands.size(), errors)); } } private Result getResult(int executing, String command) { - if (!cmdResult.containsKey(executing)) { - cmdResult.put(executing, new Result(command)); - } + return cmdResult.compute(executing, (i, r) -> { + if (r == null) { + r = new Result(command); + } else { + if (!command.equals("") && r.getCommand().equals("")) { + r.setCommand(command); + } + } - Result existing = cmdResult.get(executing); - - if (!command.equals("") && existing.getCommand().equals("")) { - existing.setCommand(command); - } - - return existing; + return r; + }); } private void logMessage(String msg) { @@ -216,16 +177,11 @@ public class Importer { } } - private static class FakeSender implements Sender { - private final Importer instance; - - private FakeSender(Importer instance) { - this.instance = instance; - } + private class FakeSender implements Sender { @Override public LuckPermsPlugin getPlatform() { - return instance.commandManager.getPlugin(); + return commandManager.getPlugin(); } @Override @@ -240,12 +196,12 @@ public class Importer { @Override public void sendMessage(String s) { - instance.logMessage(s); + logMessage(s); } @Override public void sendMessage(FancyMessage message) { - instance.logMessage(message.toOldMessageFormat()); + logMessage(message.toOldMessageFormat()); } @Override diff --git a/common/src/main/java/me/lucko/luckperms/common/plugin/LuckPermsPlugin.java b/common/src/main/java/me/lucko/luckperms/common/plugin/LuckPermsPlugin.java index 7d884904..8d092e6a 100644 --- a/common/src/main/java/me/lucko/luckperms/common/plugin/LuckPermsPlugin.java +++ b/common/src/main/java/me/lucko/luckperms/common/plugin/LuckPermsPlugin.java @@ -29,6 +29,7 @@ import me.lucko.luckperms.common.api.ApiProvider; import me.lucko.luckperms.common.caching.handlers.CachedStateManager; import me.lucko.luckperms.common.calculators.CalculatorFactory; import me.lucko.luckperms.common.commands.BaseCommand; +import me.lucko.luckperms.common.commands.CommandManager; import me.lucko.luckperms.common.commands.sender.Sender; import me.lucko.luckperms.common.commands.utils.Util; import me.lucko.luckperms.common.config.LuckPermsConfiguration; @@ -36,7 +37,6 @@ import me.lucko.luckperms.common.constants.Message; import me.lucko.luckperms.common.contexts.ContextManager; import me.lucko.luckperms.common.core.UuidCache; import me.lucko.luckperms.common.core.model.User; -import me.lucko.luckperms.common.data.Importer; import me.lucko.luckperms.common.debug.DebugHandler; import me.lucko.luckperms.common.locale.LocaleManager; import me.lucko.luckperms.common.managers.GroupManager; @@ -126,11 +126,11 @@ public interface LuckPermsPlugin { ApiProvider getApiProvider(); /** - * Gets the importer instance + * Gets the command manager * - * @return the importer + * @return the command manager */ - Importer getImporter(); + CommandManager getCommandManager(); /** * Gets the instance providing locale translations for the plugin diff --git a/common/src/main/java/me/lucko/luckperms/common/utils/HikariSupplier.java b/common/src/main/java/me/lucko/luckperms/common/utils/HikariSupplier.java index 8274c96e..c0dca01e 100644 --- a/common/src/main/java/me/lucko/luckperms/common/utils/HikariSupplier.java +++ b/common/src/main/java/me/lucko/luckperms/common/utils/HikariSupplier.java @@ -39,9 +39,9 @@ public class HikariSupplier implements AutoCloseable { private HikariDataSource hikari; - public void setup() { + public void setup(String poolName) { hikari = new HikariDataSource(); - hikari.setPoolName("powerfulperms-migrator-pool"); + hikari.setPoolName(poolName); hikari.setMaximumPoolSize(2); hikari.setDataSourceClassName("com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); hikari.addDataSourceProperty("serverName", address.split(":")[0]); diff --git a/default-lang.yml b/default-lang.yml index 801d569f..74e1e2f0 100644 --- a/default-lang.yml +++ b/default-lang.yml @@ -346,6 +346,7 @@ log-export-failure: "An unexpected error occured whilst writing to the file." log-export-success: "&aSuccessfully exported the log to &b{0}&a." import-already-running: "Another import process is already running. Please wait for it to finish and try again." +export-already-running: "Another export process is already running. Please wait for it to finish and try again." import-log-doesnt-exist: "Error: File {0} does not exist." import-log-not-readable: "Error: File {0} is not readable." import-log-failure: "An unexpected error occured whilst reading from the log file." diff --git a/sponge/src/main/java/me/lucko/luckperms/sponge/LPSpongePlugin.java b/sponge/src/main/java/me/lucko/luckperms/sponge/LPSpongePlugin.java index 870f20cd..7619602f 100644 --- a/sponge/src/main/java/me/lucko/luckperms/sponge/LPSpongePlugin.java +++ b/sponge/src/main/java/me/lucko/luckperms/sponge/LPSpongePlugin.java @@ -140,6 +140,7 @@ public class LPSpongePlugin implements LuckPermsPlugin { private boolean lateLoad = false; private LuckPermsScheduler scheduler; + private SpongeCommand commandManager; private LuckPermsConfiguration configuration; private SpongeUserManager userManager; private SpongeGroupManager groupManager; @@ -239,7 +240,7 @@ public class LPSpongePlugin implements LuckPermsPlugin { // register commands getLog().info("Registering commands..."); CommandManager cmdService = game.getCommandManager(); - SpongeCommand commandManager = new SpongeCommand(this); + commandManager = new SpongeCommand(this); cmdService.register(this, commandManager, "luckperms", "perms", "lp", "permissions", "perm"); // load internal managers @@ -248,7 +249,6 @@ public class LPSpongePlugin implements LuckPermsPlugin { userManager = new SpongeUserManager(this); groupManager = new SpongeGroupManager(this); trackManager = new GenericTrackManager(this); - importer = new Importer(commandManager); calculatorFactory = new SpongeCalculatorFactory(this); cachedStateManager = new CachedStateManager(this);