use murmur 3 32 instead of xxhash 64 for icon digests - reduce icon d…

…b size (mostly in memory, because on disk compressed using LZ4) and avoid loading extra classes on start-up (murmur is already used in classloader) no collisions so far GitOrigin-RevId: 1f41a238729772b4b3c4faa5de5cd1ccb9ec3f65
jjwwhmm · Nov 19, 2021 · 0429b01 · 0429b01
1 parent 22c1f1a
commit 0429b01
Show file tree

Hide file tree

Showing 57 changed files with 1,750 additions and 1,595 deletions.
diff --git a/images/src/icons/ImagesIcons.java b/images/src/icons/ImagesIcons.java
@@ -1,4 +1,4 @@
-// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package icons;
 
 import com.intellij.ui.IconManager;
@@ -11,12 +11,12 @@
  * DO NOT EDIT IT BY HAND, run "Generate icon classes" configuration instead
  */
 public final class ImagesIcons {
-  private static @NotNull Icon load(@NotNull String path, long cacheKey, int flags) {
+  private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {
     return IconManager.getInstance().loadRasterizedIcon(path, ImagesIcons.class.getClassLoader(), cacheKey, flags);
   }
-  /** 16x16 */ public static final @NotNull Icon ImagesFileType = load("org/intellij/images/icons/ImagesFileType.svg", -7237880555066875797L, 0);
-  /** 75x86 */ public static final @NotNull Icon ThumbnailBlank = load("org/intellij/images/icons/ThumbnailBlank.png", 0L, 2);
-  /** 75x82 */ public static final @NotNull Icon ThumbnailDirectory = load("org/intellij/images/icons/ThumbnailDirectory.png", 0L, 0);
-  /** 13x13 */ public static final @NotNull Icon ThumbnailToolWindow = load("org/intellij/images/icons/ThumbnailToolWindow.svg", -8140266098191581178L, 2);
-  /** 16x16 */ public static final @NotNull Icon ToggleTransparencyChessboard = load("org/intellij/images/icons/ToggleTransparencyChessboard.svg", -4565847047341614209L, 2);
+  /** 16x16 */ public static final @NotNull Icon ImagesFileType = load("org/intellij/images/icons/ImagesFileType.svg", -1643660520, 0);
+  /** 75x86 */ public static final @NotNull Icon ThumbnailBlank = load("org/intellij/images/icons/ThumbnailBlank.png", 0, 2);
+  /** 75x82 */ public static final @NotNull Icon ThumbnailDirectory = load("org/intellij/images/icons/ThumbnailDirectory.png", 0, 0);
+  /** 13x13 */ public static final @NotNull Icon ThumbnailToolWindow = load("org/intellij/images/icons/ThumbnailToolWindow.svg", 136570819, 2);
+  /** 16x16 */ public static final @NotNull Icon ToggleTransparencyChessboard = load("org/intellij/images/icons/ToggleTransparencyChessboard.svg", -1311027947, 2);
 }
diff --git a/java/openapi/src/icons/OpenapiIcons.java b/java/openapi/src/icons/OpenapiIcons.java
@@ -1,4 +1,4 @@
-// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+// Copyright 2000-2021 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package icons;
 
 import com.intellij.ui.IconManager;
@@ -11,8 +11,8 @@
  * DO NOT EDIT IT BY HAND, run "Generate icon classes" configuration instead
  */
 public final class OpenapiIcons {
-  private static @NotNull Icon load(@NotNull String path, long cacheKey, int flags) {
+  private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {
     return IconManager.getInstance().loadRasterizedIcon(path, OpenapiIcons.class.getClassLoader(), cacheKey, flags);
   }
-  /** 16x16 */ public static final @NotNull Icon RepositoryLibraryLogo = load("icons/repositoryLibraryLogo.svg", -1750344903510804378L, 0);
+  /** 16x16 */ public static final @NotNull Icon RepositoryLibraryLogo = load("icons/repositoryLibraryLogo.svg", 1063779084, 0);
 }
diff --git a/notebooks/visualization/src/icons/VisualisationIcons.java b/notebooks/visualization/src/icons/VisualisationIcons.java
@@ -10,25 +10,25 @@
  * DO NOT EDIT IT BY HAND, run "Generate icon classes" configuration instead
  */
 public final class VisualisationIcons {
-  private static @NotNull Icon load(@NotNull String path, long cacheKey, int flags) {
+  private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {
     return IconManager.getInstance().loadRasterizedIcon(path, VisualisationIcons.class.getClassLoader(), cacheKey, flags);
   }
 
   public static final class Chart {
-    /** 16x16 */ public static final @NotNull Icon ChartAreaRange = load("icons/chart/chartAreaRange.svg", -8240125604082800054L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartBar = load("icons/chart/chartBar.svg", 3518553399833102582L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartBubble = load("icons/chart/chartBubble.svg", -7436005997376963797L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartLine = load("icons/chart/chartLine.svg", -7179128587440517469L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartPie = load("icons/chart/chartPie.svg", -3472582953481794540L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartScatter = load("icons/chart/chartScatter.svg", -8780407383966386109L, 2);
-    /** 16x16 */ public static final @NotNull Icon ChartStock = load("icons/chart/chartStock.svg", 5477966691607881314L, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartAreaRange = load("icons/chart/chartAreaRange.svg", 302136443, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartBar = load("icons/chart/chartBar.svg", -691862941, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartBubble = load("icons/chart/chartBubble.svg", -2015034455, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartLine = load("icons/chart/chartLine.svg", -1170150335, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartPie = load("icons/chart/chartPie.svg", 788472050, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartScatter = load("icons/chart/chartScatter.svg", 246533207, 2);
+    /** 16x16 */ public static final @NotNull Icon ChartStock = load("icons/chart/chartStock.svg", 1831434359, 2);
   }
 
   public static final class Graphics {
-    /** 16x16 */ public static final @NotNull Icon ConstraintProportions = load("icons/graphics/constraintProportions.svg", 5583708402552630410L, 2);
+    /** 16x16 */ public static final @NotNull Icon ConstraintProportions = load("icons/graphics/constraintProportions.svg", 1385881763, 2);
   }
 
   public static final class Table {
-    /** 16x16 */ public static final @NotNull Icon Pagination = load("icons/table/pagination.svg", -8862976910242223129L, 2);
+    /** 16x16 */ public static final @NotNull Icon Pagination = load("icons/table/pagination.svg", -815492813, 2);
   }
 }
diff --git a/platform/build-scripts/icons/intellij.platform.images.build.iml b/platform/build-scripts/icons/intellij.platform.images.build.iml
@@ -22,7 +22,7 @@
     <orderEntry type="library" name="netty-buffer" level="project" />
     <orderEntry type="library" name="lz4-java" level="project" />
     <orderEntry type="library" name="batik-transcoder" level="project" />
-    <orderEntry type="library" scope="PROVIDED" name="opentelemetry" level="project" />
+    <orderEntry type="library" name="opentelemetry" level="project" />
     <orderEntry type="module" module-name="intellij.platform.testFramework" scope="PROVIDED" />
     <orderEntry type="module" module-name="intellij.platform.util.xmlDom" scope="PROVIDED" />
     <orderEntry type="library" name="okhttp" level="project" />

diff --git a/platform/build-scripts/icons/src/org/jetbrains/intellij/build/images/IconsClassGenerator.kt b/platform/build-scripts/icons/src/org/jetbrains/intellij/build/images/IconsClassGenerator.kt
@@ -3,17 +3,16 @@
 package org.jetbrains.intellij.build.images
 
 import com.intellij.openapi.util.text.StringUtil
-import com.intellij.ui.svg.SvgCacheManager
 import com.intellij.ui.svg.SvgTranscoder
 import com.intellij.ui.svg.createSvgDocument
 import com.intellij.util.LineSeparator
 import com.intellij.util.containers.CollectionFactory
 import com.intellij.util.containers.ContainerUtil
 import com.intellij.util.diff.Diff
+import com.intellij.util.io.Murmur3_32Hash
 import com.intellij.util.io.directoryStreamIfExists
 import com.intellij.util.io.systemIndependentPath
 import com.intellij.util.readXmlAsModel
-import net.jpountz.xxhash.XXHashFactory
 import org.jetbrains.jps.model.JpsSimpleElement
 import org.jetbrains.jps.model.java.JavaResourceRootType
 import org.jetbrains.jps.model.java.JavaSourceRootProperties
@@ -280,7 +279,7 @@ internal open class IconsClassGenerator(private val projectHome: Path,
     }
     result.append(" class ").append(info.className).append(" {\n")
     if (info.customLoad) {
-      append(result, "private static @NotNull Icon load(@NotNull String path, long cacheKey, int flags) {", 1)
+      append(result, "private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {", 1)
       append(result, "return $iconLoaderCode.loadRasterizedIcon(path, ${info.className}.class.getClassLoader(), cacheKey, flags);", 2)
       append(result, "}", 1)
 
@@ -438,7 +437,7 @@ internal open class IconsClassGenerator(private val projectHome: Path,
     }
 
     var javaDoc: String
-    var key: Long
+    var key: Int
     try {
       val loadedImage: BufferedImage
       if (file.toString().endsWith(".svg")) {
@@ -467,7 +466,7 @@ internal open class IconsClassGenerator(private val projectHome: Path,
     val relativePath = rootPrefix + rootDir.relativize(imageFile).systemIndependentPath
     assert(relativePath.startsWith("/"))
     append(result, "${javaDoc}public static final @NotNull Icon $iconName = " +
-                   "$method(\"${relativePath.removePrefix("/")}\", ${key}L, ${image.getFlags()});", level)
+                   "$method(\"${relativePath.removePrefix("/")}\", $key, ${image.getFlags()});", level)
 
     val oldName = deprecatedIconFieldNameMap.get(iconName)
     if (oldName != null) {
@@ -634,16 +633,14 @@ private fun capitalize(name: String): String {
 }
 
 private const val iconLoaderCode = "IconManager.getInstance()"
-private val hashFactory: XXHashFactory = XXHashFactory.fastestJavaInstance()
 
 // grid-layout.svg duplicates grid-view.svg, but grid-layout_dark.svg differs from grid-view_dark.svg
 // so, add filename to image id to support such scenario
-internal fun getImageKey(fileData: ByteArray, fileName: String): Long {
-  val h = hashFactory.newStreamingHash64(SvgCacheManager.HASH_SEED)
-  h.update(fileData, 0, fileData.size)
-  val nameBytes = fileName.toByteArray()
-  h.update(nameBytes, 0, nameBytes.size)
-  return h.value
+internal fun getImageKey(fileData: ByteArray, fileName: String): Int {
+  val h = Murmur3_32Hash.Murmur3_32Hasher(0)
+  h.putBytes(fileData, 0, fileData.size)
+  h.putString(fileName)
+  return h.hash()
 }
 
 // remove line separators to unify line separators (\n vs \r\n), trim lines

diff --git a/platform/build-scripts/icons/src/org/jetbrains/intellij/build/images/ImageSvgPreCompiler.kt b/platform/build-scripts/icons/src/org/jetbrains/intellij/build/images/ImageSvgPreCompiler.kt
@@ -16,7 +16,7 @@ import org.jetbrains.jps.model.java.JpsJavaExtensionService
 import org.jetbrains.jps.model.module.JpsModule
 import org.jetbrains.mvstore.MVMap
 import org.jetbrains.mvstore.MVStore
-import org.jetbrains.mvstore.type.LongDataType
+import org.jetbrains.mvstore.type.IntDataType
 import java.awt.image.BufferedImage
 import java.nio.file.Files
 import java.nio.file.NoSuchFileException
@@ -57,7 +57,7 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
 
   private val totalFiles = AtomicLong(0)
 
-  private val collisionGuard = ConcurrentHashMap<Long, FileInfo>()
+  private val collisionGuard = ConcurrentHashMap<Int, FileInfo>()
 
   companion object {
     @JvmStatic
@@ -114,17 +114,18 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
   fun compileIcons(dbFile: Path, dirs: List<Path>) {
     val storeBuilder = MVStore.Builder()
       .autoCommitBufferSize(128_1024)
+      .keysPerPage(257)
       .backgroundExceptionHandler { e, _ -> throw e }
       // fast lz4 - 14 MB, high compression - 9 MB, so, we use high even if it consumes a lot of CPU (anyway, performed in parallel)
       .compressionLevel(2)
     val store = storeBuilder.truncateAndOpen(dbFile)
     try {
-      val mapBuilder = MVMap.Builder<Long, ImageValue>()
-      mapBuilder.keyType(LongDataType.INSTANCE)
+      val mapBuilder = MVMap.Builder<Int, ImageValue>()
+      mapBuilder.keyType(IntDataType.INSTANCE)
       mapBuilder.valueType(ImageValue.ImageValueSerializer())
 
-      val scaleToMap = ConcurrentHashMap<Float, MVMap<Long, ImageValue>>(scales.size * 2, 0.75f, 2)
-      val getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Long, ImageValue> = { k, isDark ->
+      val scaleToMap = ConcurrentHashMap<Float, MVMap<Int, ImageValue>>(scales.size * 2, 0.75f, 2)
+      val getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Int, ImageValue> = { k, isDark ->
         SvgCacheManager.getMap(k, isDark, scaleToMap, store, mapBuilder)
       }
 
@@ -153,7 +154,7 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
                          rootDir: Path,
                          level: Int,
                          rootRobotData: IconRobotsData,
-                         getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Long, ImageValue>) {
+                         getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Int, ImageValue>) {
     val stream = try {
       Files.newDirectoryStream(dir)
     }
@@ -209,7 +210,7 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
   }
 
   private fun processImage(variants: List<Path>,
-                           getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Long, ImageValue>,
+                           getMapByScale: (scale: Float, isDark: Boolean) -> MutableMap<Int, ImageValue>,
                            // just to reuse
                            dimension: ImageLoader.Dimension2DDouble) {
     //println("$id: ${variants.joinToString { rootDir.relativize(it).toString() }}")
@@ -231,7 +232,6 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
     // key is the same for all variants
     val light1xBytes = light1xData.toByteArray()
     val imageKey = getImageKey(light1xBytes, light1x.fileName.toString())
-
     if (checkCollision(imageKey, light1x, light1xBytes)) {
       return
     }
@@ -251,7 +251,7 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
     }
   }
 
-  private fun checkCollision(imageKey: Long, file: Path, fileNormalizedData: ByteArray): Boolean {
+  private fun checkCollision(imageKey: Int, file: Path, fileNormalizedData: ByteArray): Boolean {
     val duplicate = collisionGuard.putIfAbsent(imageKey, FileInfo(file))
     if (duplicate == null) {
       return false
@@ -276,11 +276,11 @@ internal class ImageSvgPreCompiler(private val compilationOutputRoot: Path? = nu
   }
 }
 
-private fun addEntry(map: MutableMap<Long, ImageValue>,
+private fun addEntry(map: MutableMap<Int, ImageValue>,
                      image: BufferedImage,
                      dimension: ImageLoader.Dimension2DDouble,
                      file: Path,
-                     imageKey: Long) {
+                     imageKey: Int) {
   val newValue = SvgCacheManager.writeImage(image, dimension)
   //println("put ${(map as MVMap).id} $file : $imageKey")
   val oldValue = map.putIfAbsent(imageKey, newValue)

diff --git a/platform/collaboration-tools/gen/icons/CollaborationToolsIcons.java b/platform/collaboration-tools/gen/icons/CollaborationToolsIcons.java
@@ -11,23 +11,23 @@
  * DO NOT EDIT IT BY HAND, run "Generate icon classes" configuration instead
  */
 public final class CollaborationToolsIcons {
-  private static @NotNull Icon load(@NotNull String path, long cacheKey, int flags) {
+  private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {
     return IconManager.getInstance().loadRasterizedIcon(path, CollaborationToolsIcons.class.getClassLoader(), cacheKey, flags);
   }
-  /** 16x16 */ public static final @NotNull Icon AddEmoji = load("icons/addEmoji.svg", -5631839384967611700L, 0);
-  /** 16x16 */ public static final @NotNull Icon AddEmojiHovered = load("icons/addEmojiHovered.svg", 7914118175895354084L, 0);
-  /** 16x16 */ public static final @NotNull Icon Branch = load("icons/branch.svg", 7013179854221709094L, 2);
-  /** 16x16 */ public static final @NotNull Icon Comment = load("icons/comment.svg", -3056595990209732615L, 0);
-  /** 16x16 */ public static final @NotNull Icon CommentHovered = load("icons/commentHovered.svg", 7093337652238115394L, 0);
-  /** 16x16 */ public static final @NotNull Icon CommentReadMany = load("icons/commentReadMany.svg", -8146519964560751992L, 2);
-  /** 16x16 */ public static final @NotNull Icon CommentUnread = load("icons/commentUnread.svg", 579344038052724333L, 0);
-  /** 16x16 */ public static final @NotNull Icon CommentUnreadMany = load("icons/commentUnreadMany.svg", -1636800867176678969L, 2);
-  /** 16x16 */ public static final @NotNull Icon CommentUnresolved = load("icons/commentUnresolved.svg", -3228706837827983826L, 0);
-  /** 16x16 */ public static final @NotNull Icon Delete = load("icons/delete.svg", -7096832826219323469L, 0);
-  /** 16x16 */ public static final @NotNull Icon DeleteHovered = load("icons/deleteHovered.svg", 6938120386099586696L, 0);
-  /** 16x16 */ public static final @NotNull Icon FileUnread = load("icons/fileUnread.svg", -5535319691994743080L, 0);
-  /** 16x16 */ public static final @NotNull Icon PullRequestClosed = load("icons/pullRequestClosed.svg", -4139963041270704048L, 0);
-  /** 16x16 */ public static final @NotNull Icon PullRequestOpen = load("icons/pullRequestOpen.svg", -5145211218350150377L, 0);
-  /** 16x16 */ public static final @NotNull Icon Send = load("icons/send.svg", 4194848673382143948L, 0);
-  /** 16x16 */ public static final @NotNull Icon SendHovered = load("icons/sendHovered.svg", 6854511407842700178L, 0);
+  /** 16x16 */ public static final @NotNull Icon AddEmoji = load("icons/addEmoji.svg", -1498019086, 0);
+  /** 16x16 */ public static final @NotNull Icon AddEmojiHovered = load("icons/addEmojiHovered.svg", -203239088, 0);
+  /** 16x16 */ public static final @NotNull Icon Branch = load("icons/branch.svg", -1729430544, 2);
+  /** 16x16 */ public static final @NotNull Icon Comment = load("icons/comment.svg", -744086647, 0);
+  /** 16x16 */ public static final @NotNull Icon CommentHovered = load("icons/commentHovered.svg", 1490465624, 0);
+  /** 16x16 */ public static final @NotNull Icon CommentReadMany = load("icons/commentReadMany.svg", 1091362783, 2);
+  /** 16x16 */ public static final @NotNull Icon CommentUnread = load("icons/commentUnread.svg", -1937473850, 0);
+  /** 16x16 */ public static final @NotNull Icon CommentUnreadMany = load("icons/commentUnreadMany.svg", 1543492556, 2);
+  /** 16x16 */ public static final @NotNull Icon CommentUnresolved = load("icons/commentUnresolved.svg", -3756667, 0);
+  /** 16x16 */ public static final @NotNull Icon Delete = load("icons/delete.svg", -103361912, 0);
+  /** 16x16 */ public static final @NotNull Icon DeleteHovered = load("icons/deleteHovered.svg", 540375735, 0);
+  /** 16x16 */ public static final @NotNull Icon FileUnread = load("icons/fileUnread.svg", 1594372685, 0);
+  /** 16x16 */ public static final @NotNull Icon PullRequestClosed = load("icons/pullRequestClosed.svg", -1621538976, 0);
+  /** 16x16 */ public static final @NotNull Icon PullRequestOpen = load("icons/pullRequestOpen.svg", 1836395508, 0);
+  /** 16x16 */ public static final @NotNull Icon Send = load("icons/send.svg", 1555154758, 0);
+  /** 16x16 */ public static final @NotNull Icon SendHovered = load("icons/sendHovered.svg", -2005907271, 0);
 }