feat: AI-powered features (translation, summary, reformulation, suggested replies)

gradle.properties

@@ -8,7 +8,7 @@
 # The setting is particularly useful for tweaking memory settings.
 
 # Build Time Optimizations
-org.gradle.jvmargs=-Xmx4g -Xms512M -XX:MaxPermSize=2048m -XX:MaxMetaspaceSize=1g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:+UseParallelGC
+org.gradle.jvmargs=-Xmx4g -Xms512M -XX:MaxMetaspaceSize=1g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:+UseParallelGC
 org.gradle.configureondemand=true
 org.gradle.parallel=true
 org.gradle.vfs.watch=true

vector/build.gradle

@@ -140,12 +140,17 @@ dependencies {
     implementation "androidx.media:media:1.6.0"
     implementation "androidx.transition:transition:1.4.1"
     implementation libs.androidx.biometric
+    implementation "androidx.security:security-crypto:1.1.0-alpha06"
 
     api "org.threeten:threetenbp:1.4.0:no-tzdb"
     api "com.gabrielittner.threetenbp:lazythreetenbp:0.16.0"
 
     implementation libs.squareup.moshi
     kapt libs.squareup.moshiKotlin
+    implementation libs.squareup.retrofit
+    implementation libs.squareup.retrofitMoshi
+    implementation platform("com.squareup.okhttp3:okhttp-bom:4.11.0")
+    implementation 'com.squareup.okhttp3:okhttp'
 
     // Lifecycle
     implementation libs.androidx.lifecycleLivedata

vector/src/main/AndroidManifest.xml

@@ -53,6 +53,10 @@
     <!-- For ScreenCaptureAndroidService -->
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
 
+    <permission
+        android:name="im.vector.app.permission.APP_FUNCTIONS"
+        android:protectionLevel="signature" />
+
     <!-- Jitsi SDK is now API26+ -->
     <uses-sdk tools:overrideLibrary="com.worklets,com.swmansion.gesturehandler,org.jitsi.meet.sdk,com.oney.WebRTCModule,com.learnium.RNDeviceInfo,com.reactnativecommunity.asyncstorage,com.ocetnik.timer,com.calendarevents,com.reactnativecommunity.netinfo,com.kevinresol.react_native_default_preference,com.rnimmersive,com.rnimmersivemode,com.corbt.keepawake,com.BV.LinearGradient,com.horcrux.svg,com.oblador.performance,com.reactnativecommunity.slider,com.brentvatne.react,com.reactnativecommunity.clipboard,com.swmansion.gesturehandler.react,org.linusu,org.reactnative.maskedview,com.reactnativepagerview,com.swmansion.reanimated,com.th3rdwave.safeareacontext,com.swmansion.rnscreens,org.devio.rn.splashscreen,com.reactnativecommunity.webview,org.wonday.orientation,com.sayem.keepawake,com.splashview,com.facebook.react,com.facebook.hermes" />
 
@@ -415,6 +419,21 @@
             android:foregroundServiceType="microphone"
             android:permission="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
 
+        <!-- AppFunctions Service for AI agent inter-app orchestration -->
+        <service
+            android:name=".features.appfunctions.ElementAppFunctionService"
+            android:exported="true"
+            android:permission="im.vector.app.permission.APP_FUNCTIONS">
+            <intent-filter>
+                <action android:name="im.vector.app.ACTION_SEARCH_MESSAGES" />
+                <action android:name="im.vector.app.ACTION_READ_MESSAGES" />
+                <action android:name="im.vector.app.ACTION_SEND_MESSAGE" />
+                <action android:name="im.vector.app.ACTION_LIST_ROOMS" />
+                <action android:name="im.vector.app.ACTION_SUMMARIZE_ROOM" />
+                <action android:name="im.vector.app.ACTION_UNREAD_SUMMARY" />
+            </intent-filter>
+        </service>
+
         <!-- Receivers -->
 
         <receiver

vector/src/main/java/im/vector/app/features/ai/notificationsummary/NotificationSummaryBottomSheet.kt

@@ -0,0 +1,166 @@
+/*
+ * Copyright 2024 New Vector Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package im.vector.app.features.ai.notificationsummary
+
+import android.os.Bundle
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import android.widget.Button
+import android.widget.LinearLayout
+import android.widget.ProgressBar
+import android.widget.ScrollView
+import android.widget.TextView
+import androidx.core.os.bundleOf
+import androidx.core.view.isVisible
+import androidx.fragment.app.FragmentManager
+import androidx.lifecycle.lifecycleScope
+import com.google.android.material.bottomsheet.BottomSheetDialogFragment
+import dagger.hilt.android.AndroidEntryPoint
+import im.vector.app.R
+import kotlinx.coroutines.launch
+import timber.log.Timber
+import javax.inject.Inject
+
+@AndroidEntryPoint
+class NotificationSummaryBottomSheet : BottomSheetDialogFragment() {
+
+    @Inject lateinit var notificationSummaryService: NotificationSummaryService
+
+    private var notificationsData: ArrayList<String> = arrayListOf()
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        notificationsData = arguments?.getStringArrayList(ARG_NOTIFICATIONS) ?: arrayListOf()
+    }
+
+    override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
+        val context = requireContext()
+        val padding = (16 * context.resources.displayMetrics.density).toInt()
+
+        val root = LinearLayout(context).apply {
+            orientation = LinearLayout.VERTICAL
+            setPadding(padding, padding, padding, padding)
+        }
+
+        val title = TextView(context).apply {
+            text = getString(R.string.notification_summary_title)
+            textSize = 18f
+            setPadding(0, 0, 0, padding)
+        }
+        root.addView(title)
+
+        val progressBar = ProgressBar(context).apply {
+            isIndeterminate = true
+        }
+        root.addView(progressBar)
+
+        val statusText = TextView(context).apply {
+            text = getString(R.string.notification_summary_loading)
+        }
+        root.addView(statusText)
+
+        val scrollView = ScrollView(context).apply {
+            isVisible = false
+            layoutParams = LinearLayout.LayoutParams(
+                    LinearLayout.LayoutParams.MATCH_PARENT,
+                    (250 * context.resources.displayMetrics.density).toInt()
+            )
+        }
+
+        val summaryText = TextView(context).apply {
+            textSize = 14f
+            setPadding(0, padding / 2, 0, padding / 2)
+        }
+        scrollView.addView(summaryText)
+        root.addView(scrollView)
+
+        val buttonsLayout = LinearLayout(context).apply {
+            orientation = LinearLayout.HORIZONTAL
+            isVisible = false
+        }
+
+        val dismissButton = Button(context).apply {
+            text = getString(R.string.notification_summary_dismiss)
+            setOnClickListener {
+                parentFragmentManager.setFragmentResult(
+                        REQUEST_KEY,
+                        bundleOf(RESULT_ACTION to ACTION_DISMISS)
+                )
+                dismissAllowingStateLoss()
+            }
+        }
+        buttonsLayout.addView(dismissButton)
+
+        val markReadButton = Button(context).apply {
+            text = getString(R.string.notification_summary_mark_read)
+            setOnClickListener {
+                parentFragmentManager.setFragmentResult(
+                        REQUEST_KEY,
+                        bundleOf(RESULT_ACTION to ACTION_MARK_READ)
+                )
+                dismissAllowingStateLoss()
+            }
+        }
+        buttonsLayout.addView(markReadButton)
+        root.addView(buttonsLayout)
+
+        // Parse notifications as pairs of room:message
+        val notifPairs = notificationsData.map { line ->
+            val parts = line.split("|", limit = 2)
+            if (parts.size == 2) parts[0] to parts[1] else "" to line
+        }
+
+        Timber.d("NotificationSummaryBottomSheet: generating summary for ${notifPairs.size} notifications")
+
+        lifecycleScope.launch {
+            try {
+                val result = notificationSummaryService.generateSummary(notifPairs)
+                progressBar.isVisible = false
+                if (result != null) {
+                    statusText.isVisible = false
+                    summaryText.text = result
+                    scrollView.isVisible = true
+                    buttonsLayout.isVisible = true
+                    Timber.d("NotificationSummaryBottomSheet: summary generated successfully")
+                } else {
+                    statusText.text = getString(R.string.notification_summary_failed)
+                    Timber.d("NotificationSummaryBottomSheet: summary generation returned null")
+                }
+            } catch (e: Exception) {
+                Timber.e(e, "NotificationSummaryBottomSheet: summary generation failed")
+                progressBar.isVisible = false
+                statusText.text = getString(R.string.notification_summary_failed)
+            }
+        }
+
+        return root
+    }
+
+    companion object {
+        private const val ARG_NOTIFICATIONS = "arg_notifications"
+        private const val TAG = "NotificationSummaryBottomSheet"
+        const val REQUEST_KEY = "notification_summary_result"
+        const val RESULT_ACTION = "action"
+        const val ACTION_DISMISS = "dismiss"
+        const val ACTION_MARK_READ = "mark_read"
+
+        fun show(
+                fragmentManager: FragmentManager,
+                notifications: List<String>
+        ) {
+            // Avoid showing multiple instances
+            if (fragmentManager.findFragmentByTag(TAG) != null) return
+
+            val sheet = NotificationSummaryBottomSheet().apply {
+                arguments = Bundle().apply { putStringArrayList(ARG_NOTIFICATIONS, ArrayList(notifications)) }
+            }
+            sheet.show(fragmentManager, TAG)
+        }
+    }
+}

vector/src/main/java/im/vector/app/features/ai/notificationsummary/NotificationSummaryService.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2024 New Vector Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+package im.vector.app.features.ai.notificationsummary
+
+import im.vector.app.features.translation.TimelineTranslationManager
+import im.vector.app.features.translation.TranslateConfig
+import im.vector.app.features.translation.TranslationService
+import timber.log.Timber
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+class NotificationSummaryService @Inject constructor(
+        private val translationService: TranslationService,
+        private val config: TranslateConfig,
+        private val timelineTranslationManager: TimelineTranslationManager
+) {
+    /**
+     * Generate a summary of missed notifications using the LLM.
+     * For each notification, checks if a translation exists in cache and uses it if available.
+     * The summary is generated in French.
+     *
+     * @param notificationTexts list of pairs (roomName, messageBody)
+     * @return the generated summary, or null if generation failed or feature is disabled
+     */
+    suspend fun generateSummary(notificationTexts: List<Pair<String, String>>): String? {
+        if (!config.enabled || !config.notificationSummaryEnabled) return null
+        if (notificationTexts.isEmpty()) return null
+
+        val formatted = notificationTexts.joinToString("\n") { (room, message) ->
+            // Check if a translation exists in cache for this message body
+            val translatedMessage = timelineTranslationManager.getCachedTranslation(message)
+            val textToUse = translatedMessage ?: message
+            "[$room] $textToUse"
+        }
+
+        Timber.d("NotificationSummary: generating summary for ${notificationTexts.size} notifications")
+
+        val language = config.targetLanguage
+
+        return try {
+            translationService.complete(
+                    "Summarize the following missed notifications concisely in $language. " +
+                            "Group by room. Mention direct mentions, decisions, and pending questions.",
+                    formatted
+            )
+        } catch (e: Exception) {
+            Timber.e(e, "Notification summary generation failed")
+            null
+        }
+    }
+}