feat!: EXPOSED-320 Many-to-many relation with extra columns

documentation-website/Writerside/topics/Breaking-Changes.md

@@ -27,6 +27,22 @@
 * The transformation of a nullable column (`Column<Unwrapped?>.transform()`) requires handling null values.
   This enables conversions from `null` to a non-nullable value, and vice versa.
 * In H2 the definition of json column with default value changed from `myColumn JSON DEFAULT '{"key": "value"}'` to `myColumn JSON DEFAULT JSON '{"key": "value"}'`
+* Additional columns from intermediate tables (defined for use with DAO `via()` for many-to-many relations) are no longer ignored on batch insert of references.
+  These columns are now included and, unless column defaults are defined, values will be required when setting references by passing `InnerTableLinkEntity` instances.
+
+To continue to ignore these columns, use the non-infix version of `via()` and provide an empty list to `additionalColumns` (or a list of specific columns to include):
+```kotlin
+// given an intermediate table StarWarsFilmActors with extra columns that should be ignored
+class StarWarsFilm(id: EntityID<Int>) : IntEntity(id) {
+    companion object : IntEntityClass<StarWarsFilm>(StarWarsFilms)
+    // ...
+    var actors by Actor.via(
+        sourceColumn = StarWarsFilmActors.starWarsFilm,
+        targetColumn = StarWarsFilmActors.actor,
+        additionalColumns = emptyList()
+    )
+}
+```
 
 ## 0.54.0
 

documentation-website/Writerside/topics/Deep-Dive-into-DAO.md

@@ -321,13 +321,13 @@ class User(id: EntityID<Int>) : IntEntity(id) {
 
 ### Many-to-many reference
 In some cases, a many-to-many reference may be required.
-Let's assume you want to add a reference to the following Actors table to the StarWarsFilm class:
+Assuming that you want to add a reference to the following `Actors` table to the previous `StarWarsFilm` class:
 ```kotlin
-object Actors: IntIdTable() {
+object Actors : IntIdTable() {
     val firstname = varchar("firstname", 50)
     val lastname = varchar("lastname", 50)
 }
-class Actor(id: EntityID<Int>): IntEntity(id) {
+class Actor(id: EntityID<Int>) : IntEntity(id) {
     companion object : IntEntityClass<Actor>(Actors)
     var firstname by Actors.firstname
     var lastname by Actors.lastname
@@ -345,21 +345,96 @@ Add a reference to `StarWarsFilm`:
 ```kotlin
 class StarWarsFilm(id: EntityID<Int>) : IntEntity(id) {
     companion object : IntEntityClass<StarWarsFilm>(StarWarsFilms)
-    ...
+    // ...
     var actors by Actor via StarWarsFilmActors
-    ...
+    // ...
 }
 ```
-Note: You can set up IDs manually inside a transaction like this:
+Note: You can set up IDs manually inside a transaction and set all referenced actors like this:
 ```kotlin
 transaction {
-    // only works with UUIDTable and UUIDEntity
-    StarWarsFilm.new (UUID.randomUUID()){
-        ...
-        actors = SizedCollection(listOf(actor))
+    StarWarsFilm.new(421) {
+        // ...
+        actors = SizedCollection(actor1, actor2)
     }
 }
 ```
+Now you can access all actors (and their fields) for a `StarWarsFilm` object, `film`, in the same way you would get any other field:
+```kotlin
+film.actors.first() // returns an Actor object
+film.actors.map { it.lastname } // returns a List<String>
+```
+If the intermediate table is defined with more than just the two reference columns, these additional columns can also be accessed by
+calling `via()` on a special wrapping entity class, `InnerTableLinkEntity`, as shown below.
+
+Given a `StarWarsFilmActors` table with the extra column `roleName`:
+```kotlin
+object StarWarsFilmActors : Table() {
+    val starWarsFilm = reference("starWarsFilm", StarWarsFilms)
+    val actor = reference("actor", Actors)
+    val roleName = varchar("role_name", 64)
+    override val primaryKey = PrimaryKey(starWarsFilm, actor)
+}
+```
+The extra value stored can be accessed through an object that holds both the child entity and the additional data.
+To both allow this and still take advantage of the underlying DAO cache, a new entity class has to be defined using `InnerTableLinkEntity`,
+which details how to get and set the additional column values from the intermediate table through two overrides:
+```kotlin
+class ActorWithRole(
+    val actor: Actor,
+    val roleName: String
+) : InnerTableLinkEntity<Int>(actor) {
+    override fun getInnerTableLinkValue(column: Column<*>): Any = when (column) {
+        StarWarsFilmActors.roleName -> roleName
+        else -> error("Column does not exist in intermediate table")
+    }
+
+    companion object : InnerTableLinkEntityClass<Int, ActorWithRole>(Actors) {
+        override fun createInstance(entityId: EntityID<Int>, row: ResultRow?) = row?.let {
+            ActorWithRole(Actor.wrapRow(it), it[StarWarsFilmActors.roleName])
+        } ?: ActorWithRole(Actor(entityId), "")
+    }
+}
+```
+The original entity class reference now looks like this:
+```kotlin
+class StarWarsFilm(id: EntityID<Int>) : IntEntity(id) {
+    companion object : IntEntityClass<StarWarsFilm>(StarWarsFilms)
+    // ...
+    var actors by ActorWithRole via StarWarsFilmActors
+}
+class Actor(id: EntityID<Int>) : IntEntity(id) {
+    companion object : IntEntityClass<Actor>(Actors)
+    var firstname by Actors.firstname
+    var lastname by Actors.lastname
+}
+```
+This extra value can then be set by providing a new `ActorWithRole` instance to the parent entity field, and accessed as before:
+```kotlin
+film.actors = SizedCollection(
+    ActorWithRole(actor1, "Han Solo"),
+    ActorWithRole(actor2, "Ben Solo")
+)
+
+StarWarsFilm.all().first.actors.map { it.roleName }
+```
+<note>
+If only some additional columns in the intermediate table should be used during batch insert, these can be specified by using
+<code>via()</code> with an argument provided to <code>additionalColumns</code>:
+<code-block>
+class StarWarsFilm(id: EntityID&lt;Int&gt;) : IntEntity(id) {
+    companion object : IntEntityClass&lt;StarWarsFilm&gt;(StarWarsFilms)
+    // ...
+    var actors by ActorWithRole.via(
+        sourceColumn = StarWarsFilmActors.starWarsFilm,
+        targetColumn = StarWarsFilmActors.actor,
+        additionalColumns = listOf(StarWarsFilmActors.roleName)
+    )
+}
+</code-block>
+Setting this parameter to an <code>emptyList()</code> means all additional columns will be ignored.
+</note>
+
 ### Parent-Child reference
 Parent-child reference is very similar to many-to-many version, but an intermediate table contains both references to the same table.
 Let's assume you want to build a hierarchical entity which could have parents and children. Our tables and an entity mapping will look like

exposed-dao/api/exposed-dao.api

@@ -43,8 +43,9 @@ public class org/jetbrains/exposed/dao/Entity {
 	public final fun setValue (Lorg/jetbrains/exposed/sql/CompositeColumn;Lorg/jetbrains/exposed/dao/Entity;Lkotlin/reflect/KProperty;Ljava/lang/Object;)V
 	public final fun set_readValues (Lorg/jetbrains/exposed/sql/ResultRow;)V
 	public final fun storeWrittenValues ()V
-	public final fun via (Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;)Lorg/jetbrains/exposed/dao/InnerTableLink;
+	public final fun via (Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;Ljava/util/List;)Lorg/jetbrains/exposed/dao/InnerTableLink;
 	public final fun via (Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Table;)Lorg/jetbrains/exposed/dao/InnerTableLink;
+	public static synthetic fun via$default (Lorg/jetbrains/exposed/dao/Entity;Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/exposed/dao/InnerTableLink;
 }
 
 public final class org/jetbrains/exposed/dao/EntityBatchUpdate {
@@ -236,8 +237,8 @@ public abstract class org/jetbrains/exposed/dao/ImmutableEntityClass : org/jetbr
 }
 
 public final class org/jetbrains/exposed/dao/InnerTableLink : kotlin/properties/ReadWriteProperty {
-	public fun <init> (Lorg/jetbrains/exposed/sql/Table;Lorg/jetbrains/exposed/dao/id/IdTable;Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;)V
-	public synthetic fun <init> (Lorg/jetbrains/exposed/sql/Table;Lorg/jetbrains/exposed/dao/id/IdTable;Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun <init> (Lorg/jetbrains/exposed/sql/Table;Lorg/jetbrains/exposed/dao/id/IdTable;Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;Ljava/util/List;)V
+	public synthetic fun <init> (Lorg/jetbrains/exposed/sql/Table;Lorg/jetbrains/exposed/dao/id/IdTable;Lorg/jetbrains/exposed/dao/EntityClass;Lorg/jetbrains/exposed/sql/Column;Lorg/jetbrains/exposed/sql/Column;Ljava/util/List;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public final fun getSourceColumn ()Lorg/jetbrains/exposed/sql/Column;
 	public final fun getTable ()Lorg/jetbrains/exposed/sql/Table;
 	public final fun getTarget ()Lorg/jetbrains/exposed/dao/EntityClass;
@@ -251,6 +252,17 @@ public final class org/jetbrains/exposed/dao/InnerTableLink : kotlin/properties/
 	public fun setValue (Lorg/jetbrains/exposed/dao/Entity;Lkotlin/reflect/KProperty;Lorg/jetbrains/exposed/sql/SizedIterable;)V
 }
 
+public abstract class org/jetbrains/exposed/dao/InnerTableLinkEntity : org/jetbrains/exposed/dao/Entity {
+	public fun <init> (Lorg/jetbrains/exposed/dao/Entity;)V
+	public abstract fun getInnerTableLinkValue (Lorg/jetbrains/exposed/sql/Column;)Ljava/lang/Object;
+	public final fun getWrapped ()Lorg/jetbrains/exposed/dao/Entity;
+}
+
+public abstract class org/jetbrains/exposed/dao/InnerTableLinkEntityClass : org/jetbrains/exposed/dao/EntityClass {
+	public fun <init> (Lorg/jetbrains/exposed/dao/id/IdTable;)V
+	protected abstract fun createInstance (Lorg/jetbrains/exposed/dao/id/EntityID;Lorg/jetbrains/exposed/sql/ResultRow;)Lorg/jetbrains/exposed/dao/InnerTableLinkEntity;
+}
+
 public abstract class org/jetbrains/exposed/dao/IntEntity : org/jetbrains/exposed/dao/Entity {
 	public fun <init> (Lorg/jetbrains/exposed/dao/id/EntityID;)V
 }

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/Entity.kt

@@ -337,14 +337,19 @@ open class Entity<ID : Comparable<ID>>(val id: EntityID<ID>) {
      *
      * @param sourceColumn The intermediate table's reference column for the child entity class.
      * @param targetColumn The intermediate table's reference column for the parent entity class.
+     * @param additionalColumns Any additional columns from the intermediate table that should be included when inserting.
+     * If left `null`, all columns additional to the [sourceColumn] and [targetColumn] will be included in the insert
+     * statement and will require a value if defaults are not defined. Provide an empty list as an argument if all
+     * additional columns should be ignored.
      * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.NodesTable
      * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.Node
      * @sample org.jetbrains.exposed.sql.tests.shared.entities.ViaTests.NodeToNodes
      */
     fun <TID : Comparable<TID>, Target : Entity<TID>> EntityClass<TID, Target>.via(
         sourceColumn: Column<EntityID<ID>>,
-        targetColumn: Column<EntityID<TID>>
-    ) = InnerTableLink(sourceColumn.table, [email protected], this@via, sourceColumn, targetColumn)
+        targetColumn: Column<EntityID<TID>>,
+        additionalColumns: List<Column<*>>? = null
+    ) = InnerTableLink(sourceColumn.table, [email protected], this@via, sourceColumn, targetColumn, additionalColumns)
 
     /**
      * Deletes this [Entity] instance, both from the cache and from the database.

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityCache.kt

@@ -23,6 +23,9 @@ class EntityCache(private val transaction: Transaction) {
     internal val inserts = LinkedHashMap<IdTable<*>, MutableSet<Entity<*>>>()
     private val updates = LinkedHashMap<IdTable<*>, MutableSet<Entity<*>>>()
     internal val referrers = HashMap<Column<*>, MutableMap<EntityID<*>, SizedIterable<*>>>()
+    internal val innerTableLinks by lazy {
+        HashMap<Column<*>, MutableMap<EntityID<*>, MutableSet<InnerTableLinkEntity<*>>>>()
+    }
 
     /**
      * The amount of entities to store in this [EntityCache] per [Entity] class.
@@ -99,6 +102,14 @@ class EntityCache(private val transaction: Transaction) {
      */
     fun <ID : Comparable<ID>, T : Entity<ID>> findAll(f: EntityClass<ID, T>): Collection<T> = getMap(f).values as Collection<T>
 
+    internal fun <SID : Comparable<SID>, ID : Comparable<ID>, T : InnerTableLinkEntity<ID>> findInnerTableLink(
+        targetColumn: Column<EntityID<ID>>,
+        targetId: EntityID<ID>,
+        sourceId: EntityID<SID>
+    ): T? {
+        return innerTableLinks[targetColumn]?.get(sourceId)?.firstOrNull { it.id == targetId } as? T
+    }
+
     /** Stores the specified [Entity] in this [EntityCache] using its associated [EntityClass] as the key. */
     fun <ID : Comparable<ID>, T : Entity<ID>> store(f: EntityClass<ID, T>, o: T) {
         getMap(f)[o.id.value] = o
@@ -113,6 +124,14 @@ class EntityCache(private val transaction: Transaction) {
         getMap(o.klass.table)[o.id.value] = o
     }
 
+    internal fun <SID : Comparable<SID>, ID : Comparable<ID>, T : InnerTableLinkEntity<ID>> storeInnerTableLink(
+        targetColumn: Column<EntityID<ID>>,
+        sourceId: EntityID<SID>,
+        targetEntity: T
+    ) {
+        innerTableLinks.getOrPut(targetColumn) { HashMap() }.getOrPut(sourceId) { mutableSetOf() }.add(targetEntity)
+    }
+
     /** Removes the specified [Entity] from this [EntityCache] using its associated [table] as the key. */
     fun <ID : Comparable<ID>, T : Entity<ID>> remove(table: IdTable<ID>, o: T) {
         getMap(table).remove(o.id.value)
@@ -293,6 +312,7 @@ class EntityCache(private val transaction: Transaction) {
         inserts.clear()
         updates.clear()
         clearReferrersCache()
+        innerTableLinks.clear()
     }
 
     /** Clears this [EntityCache] of stored data that maps cached parent entities to their referencing child entities. */

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityClass.kt

@@ -163,6 +163,13 @@ abstract class EntityClass<ID : Comparable<ID>, out T : Entity<ID>>(
                 with(entity) { col.lookup() }?.let { referrers.remove(it as EntityID<*>) }
             }
         }
+        cache.innerTableLinks.forEach { (_, links) ->
+            links.remove(entity.id)
+
+            links.forEach { (_, targetEntities) ->
+                targetEntities.removeAll { it.wrapped == entity }
+            }
+        }
     }
 
     /** Returns a [SizedIterable] containing all entities with [EntityID] values from the provided [ids] list. */
@@ -207,6 +214,33 @@ abstract class EntityClass<ID : Comparable<ID>, out T : Entity<ID>>(
         wrapRow(it, alias)
     }
 
+    internal fun <SID : Comparable<SID>> wrapLinkRows(
+        rows: SizedIterable<ResultRow>,
+        targetColumn: Column<EntityID<ID>>,
+        sourceColumn: Column<EntityID<SID>>
+    ): SizedIterable<T> = rows mapLazy { wrapLinkRow(it, targetColumn, sourceColumn) }
+
+    private fun <SID : Comparable<SID>> wrapLinkRow(
+        row: ResultRow,
+        targetColumn: Column<EntityID<ID>>,
+        sourceColumn: Column<EntityID<SID>>
+    ): T {
+        val targetId = row[table.id]
+        val sourceId = row[sourceColumn]
+        val transaction = TransactionManager.current()
+        val entity = transaction.entityCache.findInnerTableLink(targetColumn, targetId, sourceId)
+            ?: createInstance(targetId, row).also { new ->
+                new.klass = this
+                new.db = transaction.db
+                warmCache().storeInnerTableLink(targetColumn, sourceId, new as InnerTableLinkEntity<ID>)
+            }
+        if (entity._readValues == null) {
+            entity._readValues = row
+        }
+
+        return entity
+    }
+
     /** Wraps the specified [ResultRow] data into an [Entity] instance. */
     @Suppress("MemberVisibilityCanBePrivate")
     fun wrapRow(row: ResultRow): T {

exposed-dao/src/main/kotlin/org/jetbrains/exposed/dao/EntityLifecycleInterceptor.kt

@@ -96,6 +96,7 @@ class EntityLifecycleInterceptor : GlobalStatementInterceptor {
     override fun beforeRollback(transaction: Transaction) {
         val entityCache = transaction.entityCache
         entityCache.clearReferrersCache()
+        entityCache.innerTableLinks.clear()
         entityCache.data.clear()
         entityCache.inserts.clear()
     }