Skip to content

Commit

Permalink
v12 SRF (per-chunk PDC support, no double-compression); fix bad bug c…
Browse files Browse the repository at this point in the history
…ausing chunks go poof. (InfernalSuite#91)

* Add v12, chunk pdc and extra nbt. Fix double compression on tile entities and entities. Fix horrible bug which made chunks go poof.

* Quick rebase on main

* Fix entity loading bug. (Authored by @AverageGithub)

* Fix entity loading bug. (Authored by @AverageGithub)

* Fix a bug with the extra tag being empty.

* Yeet debug logs

* Fix crash. (Authored by @AverageGithub)

* Update core/src/main/java/com/infernalsuite/aswm/serialization/anvil/AnvilWorldReader.java

Co-authored-by: Paul <[email protected]>

* Paul doesn't like var :(

* Review changes

---------

Co-authored-by: kyngs <[email protected]>
Co-authored-by: Paul <[email protected]>
  • Loading branch information
3 people authored and ComputerNerd100 committed Jan 9, 2024
1 parent 9277fc8 commit 1a281a7
Show file tree
Hide file tree
Showing 16 changed files with 519 additions and 38 deletions.
17 changes: 8 additions & 9 deletions SLIME_FORMAT
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
-------------------------------------
“Slime” file format
2 bytes - magic = 0xB10B
1 byte (ubyte) - version, current = 0x0B
1 byte (ubyte) - version, current = 0x0C
4 bytes (int) - world version (see version list below)
4 bytes (int) - compressed chunks size
4 bytes (int) - uncompressed chunks size
Expand All @@ -10,7 +10,7 @@

4 bytes (int) - compressed “extra” size
4 bytes (int) - uncompressed “extra” size
[depends] - compound tag compressed using zstd
[depends] - extra compound tag compressed using zstd (used for PDC, and/or custom data)
-------------------------------------

Custom chunk format
Expand All @@ -31,19 +31,17 @@ Custom chunk format
4 bytes (int) - heightmaps size
<array of heightmap nbt compounds>
same format as mc, uncompressed
4 bytes (int) - compressed tile entities size
4 bytes (int) - uncompressed tile entities size
4 bytes (int) - tile entities size
<array of tile entity nbt compounds>
Same format as mc
inside an nbt list named “tiles”, in a global compound, no gzip anywhere
compressed using zstd
4 bytes (int) compressed entities size
4 bytes (int) uncompressed entities size
uncompressed
4 bytes (int) entities size
<array of entity nbt compounds>
Same format as mc EXCEPT optional “CustomId”
inside an nbt list named “entities”, in a global compound
Compressed using zstd

[depends] - compound tag uncompressed (used for PDC, and/or custom data)
-------------------------------------

World version list:
Expand All @@ -68,4 +66,5 @@ Version history:
- v8: Variable biomes size
- v9: Fix issue with biomes size, causing old worlds to be corrupted
- v10: Use minecraft version id, remove legacy version artifacts
- v11: Move entities and tile entities into the chunk structure
- v11: Move entities and tile entities into the chunk structure
- v12: Add support for chunk-based PDC
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ public class SlimeFormat {
public static final byte[] SLIME_HEADER = new byte[] { -79, 11 };

/** Latest version of the SRF that SWM supports **/
public static final byte SLIME_VERSION = 11;
public static final byte SLIME_VERSION = 12;
}
13 changes: 13 additions & 0 deletions api/src/main/java/com/infernalsuite/aswm/api/world/SlimeChunk.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,17 @@ public interface SlimeChunk {
*/
List<CompoundTag> getEntities();

/**
* Returns the extra data of the chunk.
* Inside this {@link CompoundTag}
* can be stored any information to then be retrieved later, as it's
* saved alongside the chunk data.
* <br>
* <b>Beware, a compound tag under the key "ChunkBukkitValues" will be stored here.
* It is used for storing chunk-based Bukkit PDC. Do not overwrite it.</b>
*
* @return A {@link CompoundTag} containing the extra data of the chunk,
*/
CompoundTag getExtraData();

}
Original file line number Diff line number Diff line change
Expand Up @@ -381,9 +381,13 @@ private static SlimeChunk readChunk(CompoundTag compound, int worldVersion) {
sectionArray[index - minSectionY] = new SlimeChunkSectionSkeleton(/*paletteTag, blockStatesArray,*/ blockStatesTag, biomeTag, blockLightArray, skyLightArray);
}

CompoundTag extraTag = new CompoundTag("", new CompoundMap());

extraTag.getValue().put(compound.getValue().get("ChunkBukkitValues")); // Attempt to Migrate PDC from Anvil format to Slime format.

for (SlimeChunkSection section : sectionArray) {
if (section != null) { // Chunk isn't empty
return new SlimeChunkSkeleton(chunkX, chunkZ, sectionArray, heightMapsCompound, tileEntities, entities);
return new SlimeChunkSkeleton(chunkX, chunkZ, sectionArray, heightMapsCompound, tileEntities, entities, extraTag);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
import com.infernalsuite.aswm.api.world.SlimeChunkSection;
import com.infernalsuite.aswm.api.world.SlimeWorld;
import com.infernalsuite.aswm.api.world.properties.SlimePropertyMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
Expand All @@ -21,6 +23,8 @@

public class SlimeSerializer {

private static final Logger LOGGER = LoggerFactory.getLogger(SlimeSerializer.class);

public static byte[] serialize(SlimeWorld world) {
CompoundTag extraData = world.getExtraData();
SlimePropertyMap propertyMap = world.getPropertyMap();
Expand Down Expand Up @@ -78,7 +82,7 @@ static byte[] serializeChunks(SlimeWorld world, Collection<SlimeChunk> chunks) t
if (!ChunkPruner.canBePruned(world, chunk)) {
emptyChunks.add(chunk);
} else {
System.out.println("PRUNED: " + chunk);
LOGGER.info("PRUNED: " + chunk);
}
}

Expand Down Expand Up @@ -127,21 +131,28 @@ static byte[] serializeChunks(SlimeWorld world, Collection<SlimeChunk> chunks) t
ListTag<CompoundTag> tileEntitiesNbtList = new ListTag<>("tileEntities", TagType.TAG_COMPOUND, chunk.getTileEntities());
CompoundTag tileEntitiesCompound = new CompoundTag("", new CompoundMap(Collections.singletonList(tileEntitiesNbtList)));
byte[] tileEntitiesData = serializeCompoundTag(tileEntitiesCompound);
byte[] compressedTileEntitiesData = Zstd.compress(tileEntitiesData);

outStream.writeInt(compressedTileEntitiesData.length);
outStream.writeInt(tileEntitiesData.length);
outStream.write(compressedTileEntitiesData);
outStream.write(tileEntitiesData);

// Entities
ListTag<CompoundTag> entitiesNbtList = new ListTag<>("entities", TagType.TAG_COMPOUND, chunk.getEntities());
CompoundTag entitiesCompound = new CompoundTag("", new CompoundMap(Collections.singletonList(entitiesNbtList)));
byte[] entitiesData = serializeCompoundTag(entitiesCompound);
byte[] compressedEntitiesData = Zstd.compress(entitiesData);

outStream.writeInt(compressedEntitiesData.length);
outStream.writeInt(entitiesData.length);
outStream.write(compressedEntitiesData);
outStream.write(entitiesData);

// Extra Tag
{
if (chunk.getExtraData() == null) {
LOGGER.warn("Chunk at " + chunk.getX() + ", " + chunk.getZ() + " from world " + world.getName() + " has no extra data! When deserialized, this chunk will have an empty extra data tag!");
}
byte[] extra = serializeCompoundTag(chunk.getExtraData());

outStream.writeInt(extra.length);
outStream.write(extra);
}
}

return outByteStream.toByteArray();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import com.infernalsuite.aswm.api.world.SlimeWorld;
import com.infernalsuite.aswm.api.world.properties.SlimePropertyMap;
import com.infernalsuite.aswm.serialization.slime.reader.impl.v11.v11WorldFormat;
import com.infernalsuite.aswm.serialization.slime.reader.impl.v12.v12WorldFormat;
import com.infernalsuite.aswm.serialization.slime.reader.impl.v19.v1_9WorldFormat;
import com.infernalsuite.aswm.serialization.slime.reader.impl.v10.v10WorldFormat;

Expand All @@ -25,6 +26,7 @@ public class SlimeWorldReaderRegistry {
register(v1_9WorldFormat.FORMAT, 1, 2, 3, 4, 5, 6, 7, 8, 9);
register(v10WorldFormat.FORMAT, 10);
register(v11WorldFormat.FORMAT, 11);
register(v12WorldFormat.FORMAT, 12);
}

private static void register(VersionedByteSlimeWorldReader<SlimeWorld> format, int... bytes) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimeProper
}

chunkMap.put(new ChunkPos(x, z),
new SlimeChunkSkeleton(x, z, chunkSectionArray, heightMaps, new ArrayList<>(), new ArrayList<>())
new SlimeChunkSkeleton(x, z, chunkSectionArray, heightMaps, new ArrayList<>(), new ArrayList<>(), new CompoundTag("", new CompoundMap()))
);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimeProper
List<CompoundTag> serializedEntities = ((ListTag<CompoundTag>) entitiesCompound.getValue().get("entities")).getValue();

chunkMap.put(new ChunkPos(x, z),
new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities));
new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities, new CompoundTag("", new CompoundMap())));
}
return chunkMap;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
package com.infernalsuite.aswm.serialization.slime.reader.impl.v12;

import com.flowpowered.nbt.CompoundMap;
import com.flowpowered.nbt.CompoundTag;
import com.flowpowered.nbt.ListTag;
import com.flowpowered.nbt.stream.NBTInputStream;
import com.github.luben.zstd.Zstd;
import com.infernalsuite.aswm.ChunkPos;
import com.infernalsuite.aswm.api.exceptions.CorruptedWorldException;
import com.infernalsuite.aswm.api.exceptions.NewerFormatException;
import com.infernalsuite.aswm.api.loaders.SlimeLoader;
import com.infernalsuite.aswm.api.utils.NibbleArray;
import com.infernalsuite.aswm.api.world.SlimeChunk;
import com.infernalsuite.aswm.api.world.SlimeChunkSection;
import com.infernalsuite.aswm.api.world.SlimeWorld;
import com.infernalsuite.aswm.api.world.properties.SlimeProperties;
import com.infernalsuite.aswm.api.world.properties.SlimePropertyMap;
import com.infernalsuite.aswm.serialization.slime.reader.VersionedByteSlimeWorldReader;
import com.infernalsuite.aswm.skeleton.SkeletonSlimeWorld;
import com.infernalsuite.aswm.skeleton.SlimeChunkSectionSkeleton;
import com.infernalsuite.aswm.skeleton.SlimeChunkSkeleton;
import org.jetbrains.annotations.Nullable;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;

public class v12SlimeWorldDeSerializer implements VersionedByteSlimeWorldReader<SlimeWorld> {

public static final int ARRAY_SIZE = 16 * 16 * 16 / (8 / 4);

@Override
public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException {
int worldVersion = dataStream.readInt();

byte[] chunkBytes = readCompressed(dataStream);
Map<ChunkPos, SlimeChunk> chunks = readChunks(propertyMap, chunkBytes);

byte[] extraTagBytes = readCompressed(dataStream);
CompoundTag extraTag = readCompound(extraTagBytes);

SlimePropertyMap worldPropertyMap = propertyMap;
Optional<CompoundMap> propertiesMap = extraTag
.getAsCompoundTag("properties")
.map(CompoundTag::getValue);

if (propertiesMap.isPresent()) {
worldPropertyMap = new SlimePropertyMap(propertiesMap.get());
worldPropertyMap.merge(propertyMap);
}

return new SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraTag, worldPropertyMap, worldVersion);
}

private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException {
Map<ChunkPos, SlimeChunk> chunkMap = new HashMap<>();
DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes));

int chunks = chunkData.readInt();
for (int i = 0; i < chunks; i++) {
// ChunkPos
int x = chunkData.readInt();
int z = chunkData.readInt();

// Sections
int sectionAmount = slimePropertyMap.getValue(SlimeProperties.CHUNK_SECTION_MAX) - slimePropertyMap.getValue(SlimeProperties.CHUNK_SECTION_MIN) + 1;
SlimeChunkSection[] chunkSections = new SlimeChunkSection[sectionAmount];

int sectionCount = chunkData.readInt();
for (int sectionId = 0; sectionId < sectionCount; sectionId++) {

// Block Light Nibble Array
NibbleArray blockLightArray;
if (chunkData.readBoolean()) {
byte[] blockLightByteArray = new byte[ARRAY_SIZE];
chunkData.read(blockLightByteArray);
blockLightArray = new NibbleArray(blockLightByteArray);
} else {
blockLightArray = null;
}

// Sky Light Nibble Array
NibbleArray skyLightArray;
if (chunkData.readBoolean()) {
byte[] skyLightByteArray = new byte[ARRAY_SIZE];
chunkData.read(skyLightByteArray);
skyLightArray = new NibbleArray(skyLightByteArray);
} else {
skyLightArray = null;
}

// Block Data
byte[] blockStateData = new byte[chunkData.readInt()];
chunkData.read(blockStateData);
CompoundTag blockStateTag = readCompound(blockStateData);

// Biome Data
byte[] biomeData = new byte[chunkData.readInt()];
chunkData.read(biomeData);
CompoundTag biomeTag = readCompound(biomeData);

chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray);
}

// HeightMaps
byte[] heightMapData = new byte[chunkData.readInt()];
chunkData.read(heightMapData);
CompoundTag heightMaps = readCompound(heightMapData);

// Tile Entities

byte[] tileEntities = read(chunkData);

CompoundTag tileEntitiesCompound = readCompound(tileEntities);
@SuppressWarnings("unchecked")
List<CompoundTag> serializedTileEntities = ((ListTag<CompoundTag>) tileEntitiesCompound.getValue().get("tileEntities")).getValue();

// Entities

byte[] entities = read(chunkData);

CompoundTag entitiesCompound = readCompound(entities);
@SuppressWarnings("unchecked")
List<CompoundTag> serializedEntities = ((ListTag<CompoundTag>) entitiesCompound.getValue().get("entities")).getValue();


// Extra Tag
byte[] rawExtra = read(chunkData);
CompoundTag extra = readCompound(rawExtra);
// If the extra tag is empty, the serializer will save it as null.
// So if we deserialize a null extra tag, we will assume it was empty.
if (extra == null) {
extra = new CompoundTag("", new CompoundMap());
}

chunkMap.put(new ChunkPos(x, z),
new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities, extra));
}
return chunkMap;
}

private static byte[] readCompressed(DataInputStream stream) throws IOException {
int compressedLength = stream.readInt();
int decompressedLength = stream.readInt();
byte[] compressedData = new byte[compressedLength];
byte[] decompressedData = new byte[decompressedLength];
stream.read(compressedData);
Zstd.decompress(decompressedData, compressedData);
return decompressedData;
}

private static byte[] read(DataInputStream stream) throws IOException {
int length = stream.readInt();
byte[] data = new byte[length];
stream.read(data);
return data;
}

private static CompoundTag readCompound(byte[] tagBytes) throws IOException {
if (tagBytes.length == 0) {
return null;
}

NBTInputStream nbtStream = new NBTInputStream(new ByteArrayInputStream(tagBytes), NBTInputStream.NO_COMPRESSION, ByteOrder.BIG_ENDIAN);
return (CompoundTag) nbtStream.readTag();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.infernalsuite.aswm.serialization.slime.reader.impl.v12;

import com.infernalsuite.aswm.api.world.SlimeWorld;
import com.infernalsuite.aswm.serialization.slime.reader.impl.SimpleWorldFormat;

public interface v12WorldFormat {

SimpleWorldFormat<SlimeWorld> FORMAT = new SimpleWorldFormat<>(data -> data, new v12SlimeWorldDeSerializer());

}
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.infernalsuite.aswm.serialization.slime.reader.impl.v19;

import com.flowpowered.nbt.CompoundMap;
import com.flowpowered.nbt.CompoundTag;
import com.infernalsuite.aswm.ChunkPos;
import com.infernalsuite.aswm.serialization.SlimeWorldReader;
import com.infernalsuite.aswm.skeleton.SkeletonSlimeWorld;
Expand Down Expand Up @@ -61,7 +63,8 @@ public SlimeWorld readFromData(v1_9SlimeWorld data) {
sections,
slimeChunk.heightMap,
slimeChunk.tileEntities,
slimeChunk.entities
slimeChunk.entities,
new CompoundTag("", new CompoundMap())
));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ private static Map<ChunkPos, SlimeChunk> cloneChunkStorage(Collection<SlimeChunk
copied,
chunk.getHeightMaps().clone(),
deepClone(chunk.getTileEntities()),
deepClone(chunk.getEntities())
deepClone(chunk.getEntities()),
chunk.getExtraData().clone()
));
}

Expand Down
Loading

0 comments on commit 1a281a7

Please sign in to comment.