Akkariin/Grasscutter
1
0
Fork 0
mirror of https://github.com/Grasscutters/Grasscutter.git synced 2025-02-09 11:03:16 +08:00
Code Issues Projects Releases Wiki Activity

Revert Multi-threaded resource loading

Browse Source
This commit is contained in:
KingRainbow44 2023-05-20 02:55:33 -04:00
parent 1a6fa43367
commit cad6e90c90
No known key found for this signature in database
GPG Key ID: FC2CB64B00D257BE
13 changed files with 117 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/emu/grasscutter/Grasscutter.java
View File

@ -125,7 +125,7 @@ public final class Grasscutter {
}
// Initialize database.
DatabaseManager.initializeAsync();
DatabaseManager.initialize();
// Initialize the default systems.
authenticationSystem = new DefaultAuthentication();

274
src/main/java/emu/grasscutter/data/ResourceLoader.java
View File

@ -25,7 +25,6 @@ import emu.grasscutter.scripts.ScriptLoader;
import emu.grasscutter.utils.FileUtils;
import emu.grasscutter.utils.JsonUtils;
import emu.grasscutter.utils.TsvUtils;
import emu.grasscutter.utils.lang.Language;
import it.unimi.dsi.fastutil.Pair;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
@ -42,7 +41,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.regex.Pattern;
@ -100,151 +98,87 @@ public final class ResourceLoader {
return List.copyOf(map.values());
}
/**
* Runs a task asynchronously.
*
* @param task The task to run.
* @return A CompletableFuture that will complete when the task is done.
*/
public static CompletableFuture<Boolean> runAsync(Runnable task) {
return CompletableFuture.supplyAsync(
() -> {
task.run();
return true;
});
}
/**
* Waits for all futures to complete.
*
* @param futures The futures to wait for.
*/
public static void waitForAll(Collection<CompletableFuture<Boolean>> futures) {
futures.forEach(CompletableFuture::join);
}
@SneakyThrows
public static void loadAll() {
if (loadedAll) return;
loadedAll = true;
Grasscutter.getLogger().info(translate("messages.status.resources.loading"));
// Mark the starting time.
var startTime = System.nanoTime();
// Initialize the script loader.
ScriptLoader.init();
// Load 'TextMaps'.
var textMaps = ResourceLoader.runAsync(Language::loadTextMaps);
// Load 'BinOutput'.
var binOutput = ResourceLoader.loadConfigData();
// Load ability lists.
var abilities =
ResourceLoader.runAsync(
() -> {
ResourceLoader.loadAbilityEmbryos();
ResourceLoader.loadOpenConfig();
ResourceLoader.loadAbilityModifiers();
});
// Load 'ExcelBinOutput'.
var errors = new ConcurrentLinkedQueue<Pair<String, Exception>>();
var excelBinOutput = ResourceLoader.loadResources(true, errors);
// Load spawn data and quests.
var scene =
ResourceLoader.runAsync(
() -> {
ResourceLoader.loadSpawnData();
ResourceLoader.loadQuests();
ResourceLoader.loadScriptSceneData();
});
loadConfigData();
// Load ability lists
loadAbilityEmbryos();
loadOpenConfig();
loadAbilityModifiers();
// Load resources
loadResources(true);
// Process into depots
GameDepot.load();
// Load spawn data and quests
loadSpawnData();
loadQuests();
loadScriptSceneData();
// Load scene points - must be done AFTER resources are loaded
loadScenePoints();
// Load default home layout
var entities =
ResourceLoader.runAsync(
() -> {
ResourceLoader.loadHomeworldDefaultSaveData();
ResourceLoader.loadNpcBornData();
ResourceLoader.loadBlossomResources();
ResourceLoader.cacheTalentLevelSets();
});
loadHomeworldDefaultSaveData();
loadNpcBornData();
loadBlossomResources();
cacheTalentLevelSets();
// Load custom server resources.
var customServer =
ResourceLoader.runAsync(
() -> {
ResourceLoader.loadConfigLevelEntityData();
ResourceLoader.loadQuestShareConfig();
ResourceLoader.loadGadgetMappings();
ResourceLoader.loadActivityCondGroups();
ResourceLoader.loadGroupReplacements();
ResourceLoader.loadTrialAvatarCustomData();
loadConfigLevelEntityData();
loadQuestShareConfig();
loadGadgetMappings();
loadActivityCondGroups();
loadGroupReplacements();
loadTrialAvatarCustomData();
EntityControllerScriptManager.load();
});
// Wait for all futures to complete.
var futures = new ArrayList<>(List.of(textMaps, abilities, scene, entities, customServer));
futures.addAll(binOutput);
futures.addAll(excelBinOutput);
ResourceLoader.waitForAll(futures);
// Load dependent-resources.
GameDepot.load(); // Process into depots
ResourceLoader.loadScenePoints(); // Load scene points.
// Log any errors.
errors.forEach(
pair ->
Grasscutter.getLogger()
.error("Error loading resource file: " + pair.left(), pair.right() + "."));
// Calculate the ending time.
var endTime = System.nanoTime();
var ns = (endTime - startTime); // divide by 1000000 to get milliseconds.
Grasscutter.getLogger().debug("Loading resources took " + ns + "ns (" + ns / 1000000 + "ms).");
EntityControllerScriptManager.load();
Grasscutter.getLogger().info(translate("messages.status.resources.finish"));
loadedAll = true;
}
public static void loadResources() {
loadResources(false, new ConcurrentLinkedQueue<>());
loadResources(false);
}
/**
* Loads all resources from annotated classes.
*
* @param doReload Whether to reload resources.
*/
public static List<CompletableFuture<Boolean>> loadResources(
boolean doReload, Queue<Pair<String, Exception>> errors) {
// Load all resources in parallel.
return ResourceLoader.getResourceDefClassesPrioritySets().stream()
.map(
classes ->
classes.stream()
.parallel()
.unordered()
.map(
c -> {
var type = c.getAnnotation(ResourceType.class);
if (type == null) return null;
public static void loadResources(boolean doReload) {
long startTime = System.nanoTime();
val errors =
new ConcurrentLinkedQueue<
Pair<String, Exception>>(); // Logger in a parallel stream will deadlock
var map = GameData.getMapByResourceDef(c);
if (map == null) return null;
getResourceDefClassesPrioritySets()
.forEach(
classes -> {
classes.stream()
.parallel()
.unordered()
.forEach(
c -> {
val type = c.getAnnotation(ResourceType.class);
if (type == null) return;
return ResourceLoader.runAsync(
() -> {
try {
loadFromResource(c, type, map, doReload);
} catch (Exception e) {
errors.add(Pair.of(Arrays.toString(type.name()), e));
}
});
})
.toList())
.flatMap(Collection::stream)
.toList();
val map = GameData.getMapByResourceDef(c);
if (map == null) return;
try {
loadFromResource(c, type, map, doReload);
} catch (Exception e) {
errors.add(Pair.of(Arrays.toString(type.name()), e));
}
});
});
errors.forEach(
pair ->
Grasscutter.getLogger()
.error("Error loading resource file: " + pair.left(), pair.right()));
long endTime = System.nanoTime();
long ns = (endTime - startTime); // divide by 1000000 to get milliseconds.
Grasscutter.getLogger().debug("Loading resources took " + ns + "ns == " + ns / 1000000 + "ms");
}
@SuppressWarnings("rawtypes")
@ -292,38 +226,38 @@ public final class ResourceLoader {
private static void loadScenePoints() {
val pattern = Pattern.compile("scene([0-9]+)_point\\.json");
try (var stream =
Files.newDirectoryStream(getResourcePath("BinOutput/Scene/Point"), "scene*_point.json")) {
stream.forEach(
path -> {
var matcher = pattern.matcher(path.getFileName().toString());
if (!matcher.find()) return;
var sceneId = Integer.parseInt(matcher.group(1));
try {
Files.newDirectoryStream(getResourcePath("BinOutput/Scene/Point"), "scene*_point.json")
.forEach(
path -> {
val matcher = pattern.matcher(path.getFileName().toString());
if (!matcher.find()) return;
int sceneId = Integer.parseInt(matcher.group(1));
ScenePointConfig config;
try {
config = JsonUtils.loadToClass(path, ScenePointConfig.class);
} catch (Exception e) {
e.printStackTrace();
return;
}
ScenePointConfig config;
try {
config = JsonUtils.loadToClass(path, ScenePointConfig.class);
} catch (Exception e) {
e.printStackTrace();
return;
}
if (config.points == null) return;
if (config.points == null) return;
var scenePoints = new IntArrayList();
config.points.forEach(
(pointId, pointData) -> {
var scenePoint = new ScenePointEntry(sceneId, pointData);
scenePoints.add((int) pointId);
pointData.setId(pointId);
val scenePoints = new IntArrayList();
config.points.forEach(
(pointId, pointData) -> {
val scenePoint = new ScenePointEntry(sceneId, pointData);
scenePoints.add((int) pointId);
pointData.setId(pointId);
GameData.getScenePointIdList().add((int) pointId);
GameData.getScenePointEntryMap().put((sceneId << 16) + pointId, scenePoint);
GameData.getScenePointIdList().add((int) pointId);
GameData.getScenePointEntryMap().put((sceneId << 16) + pointId, scenePoint);
pointData.updateDailyDungeon();
});
GameData.getScenePointsPerScene().put(sceneId, scenePoints);
});
pointData.updateDailyDungeon();
});
GameData.getScenePointsPerScene().put(sceneId, scenePoints);
});
} catch (IOException ignored) {
Grasscutter.getLogger()
.error("Scene point files cannot be found, you cannot use teleport waypoints!");
@ -674,34 +608,12 @@ public final class ResourceLoader {
}
}
/** Loads data from parsed files. */
private static List<CompletableFuture<Boolean>> loadConfigData() {
var tasks = new ArrayList<CompletableFuture<Boolean>>();
// Load config data.
tasks.add(
ResourceLoader.runAsync(
() ->
loadConfigData(
GameData.getAvatarConfigData(),
"BinOutput/Avatar/",
ConfigEntityAvatar.class)));
tasks.add(
ResourceLoader.runAsync(
() ->
loadConfigData(
GameData.getMonsterConfigData(),
"BinOutput/Monster/",
ConfigEntityMonster.class)));
tasks.add(
ResourceLoader.runAsync(
() ->
loadConfigDataMap(
GameData.getGadgetConfigData(),
"BinOutput/Gadget/",
ConfigEntityGadget.class)));
return tasks;
private static void loadConfigData() {
loadConfigData(GameData.getAvatarConfigData(), "BinOutput/Avatar/", ConfigEntityAvatar.class);
loadConfigData(
GameData.getMonsterConfigData(), "BinOutput/Monster/", ConfigEntityMonster.class);
loadConfigDataMap(
GameData.getGadgetConfigData(), "BinOutput/Gadget/", ConfigEntityGadget.class);
}
private static <T extends ConfigEntityBase> void loadConfigData(

6
src/main/java/emu/grasscutter/database/DatabaseManager.java
View File

@ -35,12 +35,6 @@ public final class DatabaseManager {
return getGameDatastore().getDatabase();
}
/** Performs the database initialization process. This occurs on a separate thread. */
public static void initializeAsync() {
new Thread(DatabaseManager::initialize).start();
}
/** Performs the database initialization process. This method is blocking. */
public static void initialize() {
// Initialize
MongoClient gameMongoClient = MongoClients.create(DATABASE.game.connectionUri);

4
src/main/java/emu/grasscutter/game/drop/DropSystemLegacy.java
View File

@ -3,7 +3,6 @@ package emu.grasscutter.game.drop;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.data.excels.ItemData;
import emu.grasscutter.game.entity.EntityItem;
import emu.grasscutter.game.entity.EntityMonster;
@ -27,8 +26,7 @@ public class DropSystemLegacy extends BaseGameSystem {
public DropSystemLegacy(GameServer server) {
super(server);
this.dropData = new Int2ObjectOpenHashMap<>();
ResourceLoader.runAsync(this::load);
this.load();
}
public Int2ObjectMap<List<DropData>> getDropData() {

4
src/main/java/emu/grasscutter/game/expedition/ExpeditionSystem.java
View File

@ -2,7 +2,6 @@ package emu.grasscutter.game.expedition;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.server.game.BaseGameSystem;
import emu.grasscutter.server.game.GameServer;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
@ -14,9 +13,8 @@ public class ExpeditionSystem extends BaseGameSystem {
public ExpeditionSystem(GameServer server) {
super(server);
this.expeditionRewardData = new Int2ObjectOpenHashMap<>();
ResourceLoader.runAsync(this::load);
this.load();
}
public Int2ObjectMap<List<ExpeditionRewardDataList>> getExpeditionRewardDataList() {

4
src/main/java/emu/grasscutter/game/gacha/GachaSystem.java
View File

@ -6,7 +6,6 @@ import com.sun.nio.file.SensitivityWatchEventModifier;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.data.common.ItemParamData;
import emu.grasscutter.data.excels.ItemData;
import emu.grasscutter.database.DatabaseHelper;
@ -46,9 +45,8 @@ public class GachaSystem extends BaseGameSystem {
public GachaSystem(GameServer server) {
super(server);
this.gachaBanners = new Int2ObjectOpenHashMap<>();
ResourceLoader.runAsync(this::load);
this.load();
this.startWatcher(server);
}

4
src/main/java/emu/grasscutter/game/shop/ShopSystem.java
View File

@ -5,7 +5,6 @@ import static emu.grasscutter.config.Configuration.GAME_OPTIONS;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.data.common.ItemParamData;
import emu.grasscutter.data.excels.ShopGoodsData;
import emu.grasscutter.server.game.BaseGameSystem;
@ -25,10 +24,9 @@ public class ShopSystem extends BaseGameSystem {
public ShopSystem(GameServer server) {
super(server);
this.shopData = new Int2ObjectOpenHashMap<>();
this.shopChestData = new Int2ObjectOpenHashMap<>();
ResourceLoader.runAsync(this::load);
this.load();
}
public static int getShopNextRefreshTime(ShopInfo shopInfo) {

3
src/main/java/emu/grasscutter/game/systems/AnnouncementSystem.java
View File

@ -2,7 +2,6 @@ package emu.grasscutter.game.systems;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.game.player.Player;
import emu.grasscutter.game.world.World;
import emu.grasscutter.net.proto.AnnounceDataOuterClass;
@ -24,7 +23,7 @@ public class AnnouncementSystem extends BaseGameSystem {
public AnnouncementSystem(GameServer server) {
super(server);
this.announceConfigItemMap = new HashMap<>();
ResourceLoader.runAsync(this::loadConfig);
loadConfig();
}
private int loadConfig() {

4
src/main/java/emu/grasscutter/game/tower/TowerSystem.java
View File

@ -3,7 +3,6 @@ package emu.grasscutter.game.tower;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.data.excels.tower.TowerScheduleData;
import emu.grasscutter.server.game.BaseGameSystem;
import emu.grasscutter.server.game.GameServer;
@ -16,8 +15,7 @@ public class TowerSystem extends BaseGameSystem {
public TowerSystem(GameServer server) {
super(server);
ResourceLoader.runAsync(this::load);
this.load();
}
public synchronized void load() {

3
src/main/java/emu/grasscutter/game/world/WorldDataSystem.java
View File

@ -3,7 +3,6 @@ package emu.grasscutter.game.world;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.data.DataLoader;
import emu.grasscutter.data.GameData;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.data.excels.InvestigationMonsterData;
import emu.grasscutter.data.excels.RewardPreviewData;
import emu.grasscutter.data.excels.world.WorldLevelData;
@ -32,7 +31,7 @@ public class WorldDataSystem extends BaseGameSystem {
this.chestInteractHandlerMap = new HashMap<>();
this.sceneInvestigationGroupMap = new ConcurrentHashMap<>();
ResourceLoader.runAsync(this::loadChestConfig);
loadChestConfig();
}
public synchronized void loadChestConfig() {

11
src/main/java/emu/grasscutter/plugin/PluginManager.java
View File

@ -1,15 +1,13 @@
package emu.grasscutter.plugin;
import static emu.grasscutter.utils.lang.Language.translate;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.server.event.Event;
import emu.grasscutter.server.event.EventHandler;
import emu.grasscutter.server.event.HandlerPriority;
import emu.grasscutter.utils.FileUtils;
import emu.grasscutter.utils.JsonUtils;
import lombok.AllArgsConstructor;
import lombok.Getter;
import javax.annotation.Nullable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStreamReader;
@ -20,8 +18,9 @@ import java.net.URLClassLoader;
import java.util.*;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import static emu.grasscutter.utils.lang.Language.translate;
import javax.annotation.Nullable;
import lombok.AllArgsConstructor;
import lombok.Getter;
/** Manages the server's plugins and the event system. */
public final class PluginManager {

15
src/main/java/emu/grasscutter/server/game/GameServer.java
View File

@ -3,7 +3,6 @@ package emu.grasscutter.server.game;
import emu.grasscutter.GameConstants;
import emu.grasscutter.Grasscutter;
import emu.grasscutter.Grasscutter.ServerRunMode;
import emu.grasscutter.data.ResourceLoader;
import emu.grasscutter.database.DatabaseHelper;
import emu.grasscutter.game.Account;
import emu.grasscutter.game.battlepass.BattlePassSystem;
@ -143,15 +142,11 @@ public final class GameServer extends KcpServer implements Iterable<Player> {
this.init(GameSessionManager.getListener(), channelConfig, address);
// Load game managers asyncronously.
ResourceLoader.runAsync(
() -> {
EnergyManager.initialize();
StaminaManager.initialize();
CookingManager.initialize();
CookingCompoundManager.initialize();
CombineManger.initialize();
});
EnergyManager.initialize();
StaminaManager.initialize();
CookingManager.initialize();
CookingCompoundManager.initialize();
CombineManger.initialize();
// Game Server base
this.address = address;

16
src/main/java/emu/grasscutter/utils/lang/Language.java
View File

@ -342,13 +342,11 @@ public final class Language {
*/
public static void loadTextMaps(boolean bypassCache) {
// Check system timestamps on cache and resources
if (!bypassCache) {
if (!bypassCache)
try {
long cacheModified = Files.getLastModifiedTime(TEXTMAP_CACHE_PATH).toMillis();
var stream = Files.list(getResourcePath("TextMap"));
var textmapsModified =
stream
long textmapsModified =
Files.list(getResourcePath("TextMap"))
.filter(path -> path.toString().endsWith(".json"))
.map(
path -> {
@ -356,17 +354,16 @@ public final class Language {
return Files.getLastModifiedTime(path).toMillis();
} catch (Exception ignored) {
Grasscutter.getLogger()
.debug("Exception while checking modified time: {}.", path);
.debug("Exception while checking modified time: ", path);
return Long.MAX_VALUE; // Don't use cache, something has gone wrong
}
})
.max(Long::compare)
.get();
stream.close();
Grasscutter.getLogger()
.debug(
"Cache modified %d, textmap modified %d."
"Cache modified %d, textmap modified %d"
.formatted(cacheModified, textmapsModified));
if (textmapsModified < cacheModified) {
// Try loading from cache
@ -375,9 +372,8 @@ public final class Language {
return;
}
} catch (Exception exception) {
Grasscutter.getLogger().error("Error loading textmaps cache: " + exception);
Grasscutter.getLogger().error("Error loading textmaps cache: " + exception.toString());
}
}
// Regenerate cache
Grasscutter.getLogger().debug("Generating TextMaps cache");