The indexer is designed to run on-device in a coroutine Flow with no blocking I/O and aggressive cancellation. Unit tests therefore use pure in-memory test doubles rather than the real filesystem or database.
Test File System
TestFileSystem implements FileSystemInterface entirely in memory. It is the universal test double used by every indexer test. The most important method is listOrgFiles, which respects the recursive parameter correctly: non-recursive listing only returns files whose parent directory exactly matches the query path.
package computer.whatthefuck.arcology.indexer
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.datetime.Instant
/**
* Test implementation of FileSystemInterface for unit testing
*/
open class TestFileSystem : FileSystemInterface {
private val files = mutableMapOf<String, FileData>()
data class FileData(
val content: String,
val contentBytes: ByteArray? = null,
val lastModified: Instant = Instant.fromEpochSeconds(1640995200) // Default test timestamp
)
/**
* Add a file to the test file system
*/
fun addFile(path: String, content: String, lastModified: Instant = Instant.fromEpochSeconds(1640995200)) {
files[path] = FileData(content, content.encodeToByteArray(), lastModified)
}
/**
* Add a binary file to the test file system
*/
fun addBinaryFile(path: String, content: ByteArray, lastModified: Instant = Instant.fromEpochSeconds(1640995200)) {
files[path] = FileData(content.decodeToString(), content, lastModified)
}
/**
* Remove a file from the test file system
*/
fun removeFile(path: String) {
files.remove(path)
}
/**
* Clear all files
*/
fun clear() {
files.clear()
}
override suspend fun fileExists(path: String): Boolean {
return files.containsKey(path)
}
override suspend fun readFile(path: String): String {
return files[path]?.content ?: throw java.io.FileNotFoundException("File not found: $path")
}
override suspend fun readFileBytes(path: String): ByteArray {
val data = files[path] ?: throw java.io.FileNotFoundException("File not found: $path")
return data.contentBytes ?: data.content.encodeToByteArray()
}
override suspend fun writeFile(path: String, content: String) {
files[path] = FileData(content, content.encodeToByteArray(), files[path]?.lastModified ?: Instant.fromEpochSeconds(1640995200))
}
override suspend fun getLastModified(path: String): Instant {
return files[path]?.lastModified ?: throw java.io.FileNotFoundException("File not found: $path")
}
override fun listOrgFiles(path: String, recursive: Boolean): Flow<String> {
return files.keys.filter { filePath ->
if (recursive) {
// Match files that start with the path and end with .org
filePath.startsWith(path) && filePath.endsWith(".org")
} else {
// Match files directly in the path directory
val parentDir = filePath.substringBeforeLast("/", "")
parentDir == path.trimEnd('/') && filePath.endsWith(".org")
}
}.sorted().asFlow()
}
override suspend fun readIgnoreFile(rootPath: String): String? = null
override suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> {
// Compute parent directory of baseFilePath
val parentDir = baseFilePath.substringBeforeLast("/", "")
val targetPrefix = if (parentDir.isEmpty()) relativeDir else "$parentDir/$relativeDir"
// Find files that are direct children of the target directory
return files.keys.filter { path ->
path.startsWith("$targetPrefix/") &&
path.removePrefix("$targetPrefix/").contains("/").not()
}.sorted()
}
}Flow Cancellation Tests
Coroutines can be cancelled at any yield or suspension point. The indexer must handle cancellation gracefully: no dangling database entries, no infinite loops, and already-processed files must remain in the repository.
The cancellation tests use MockTestRepository and MockTestFileSystem (simpler variants of the doubles) to assert exact event counts.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.parser.OrgFileParser
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertFalse
import kotlin.test.assertTrue
/**
* Tests for Flow cancellation behavior throughout the indexing pipeline
*/
class FlowCancellationTest {
@Test
fun testCancellationDuringFileDiscovery() = runTest {
// Given: A scenario with many files to give time for cancellation
val files = (1..10).associate { i ->
"file$i.org" to ":PROPERTIES:\n:ID: test-id-$i\n:END:\n* Heading $i\n Content $i"
}
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(files)
val config = IndexingConfig(batchSize = 1)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Starting indexing and then cancelling from outside
var collectedEvents = mutableListOf<IndexProgress>()
var wasCancelled = false
val job = launch {
try {
indexer.indexDirectoryFlow("testdir", true).collect { progress ->
collectedEvents.add(progress)
}
} catch (e: CancellationException) {
wasCancelled = true
throw e
}
}
// Wait for some files to be discovered, then cancel
while (collectedEvents.count { it is IndexProgress.FileDiscovered } < 3 && job.isActive) {
delay(1)
}
job.cancel()
job.join()
// Then: Should have collected some discovery events before cancellation
assertTrue(collectedEvents.any { it is IndexProgress.FileDiscovered }, "Should have discovered some files before cancellation")
assertFalse(job.isActive, "Job should be completed (cancelled)")
}
@Test
fun testCancellationDuringFileProcessing() = runTest {
// Given: A scenario with slow file processing
val files = (1..5).associate { i ->
"file$i.org" to ":PROPERTIES:\n:ID: test-id-$i\n:END:\n* Heading $i\n Content $i"
}
val repository = MockTestRepository(simulateSlowOperations = true)
val fileSystem = MockTestFileSystem(files)
val config = IndexingConfig(batchSize = 1, fileParseTimeoutMs = 10000) // Long timeout to avoid timeout cancellation
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Starting indexing and cancelling during processing
val collectedEvents = mutableListOf<IndexProgress>()
val job = launch {
try {
indexer.indexDirectoryFlow("testdir", true).collect { progress ->
collectedEvents.add(progress)
}
} catch (e: CancellationException) {
throw e
}
}
// Wait for file processing to start, then cancel
while (collectedEvents.count { it is IndexProgress.FileProcessingStarted } < 2 && job.isActive) {
delay(1)
}
job.cancel()
job.join()
// Then: Should have started processing before cancellation
val processingStartedCount = collectedEvents.count { it is IndexProgress.FileProcessingStarted }
assertTrue(processingStartedCount >= 2, "Should have started processing multiple files")
assertTrue(collectedEvents.any { it is IndexProgress.FileProcessingStarted }, "Should have processing events")
assertFalse(job.isActive, "Job should be completed (cancelled)")
}
@Test
fun testCancellationDuringBatchProcessing() = runTest {
// Given: A scenario designed to trigger batch processing
val files = (1..6).associate { i ->
"file$i.org" to ":PROPERTIES:\n:ID: test-id-$i\n:END:\n* Heading $i\n Content $i"
}
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(files)
val config = IndexingConfig(batchSize = 3) // Will trigger batch processing
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Cancelling during batch processing
val collectedEvents = mutableListOf<IndexProgress>()
val job = launch {
try {
indexer.indexDirectoryFlow("testdir", true).collect { progress ->
collectedEvents.add(progress)
}
} catch (e: CancellationException) {
throw e
}
}
// Wait for first batch to complete, then cancel
while (collectedEvents.none { it is IndexProgress.BatchCompleted } && job.isActive) {
delay(1)
}
job.cancel()
job.join()
// Then: Should have processed at least one batch
assertTrue(collectedEvents.any { it is IndexProgress.BatchCompleted }, "Should have completed at least one batch")
assertTrue(collectedEvents.any { it is IndexProgress.FileProcessed }, "Should have processed some files")
assertFalse(job.isActive, "Job should be completed (cancelled)")
}
@Test
fun testGracefulCancellationPreservesData() = runTest {
// Given: A scenario where some files are processed before cancellation
val files = mapOf(
"file1.org" to ":PROPERTIES:\n:ID: test-id-1\n:END:\n* Heading 1\n Content 1",
"file2.org" to ":PROPERTIES:\n:ID: test-id-2\n:END:\n* Heading 2\n Content 2",
"file3.org" to ":PROPERTIES:\n:ID: test-id-3\n:END:\n* Heading 3\n Content 3"
)
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(files)
val config = IndexingConfig(batchSize = 1)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Processing files and cancelling after some are processed
val collectedEvents = mutableListOf<IndexProgress>()
val job = launch {
try {
indexer.indexDirectoryFlow("testdir", true).collect { progress ->
collectedEvents.add(progress)
}
} catch (e: CancellationException) {
throw e
}
}
// Wait for some files to be processed, then cancel
while (collectedEvents.count { it is IndexProgress.FileProcessed } < 2 && job.isActive) {
delay(1)
}
job.cancel()
job.join()
// Then: Processed files should be stored in repository
val processedCount = collectedEvents.count { it is IndexProgress.FileProcessed }
assertTrue(processedCount >= 2, "Should have processed at least 2 files")
assertTrue(repository.getStoredFiles().size >= 2, "Repository should contain processed files")
assertTrue(repository.getStoredNodes().isNotEmpty(), "Repository should contain parsed nodes")
}
@Test
fun testTimeoutCancellationVsManualCancellation() = runTest {
// Given: A scenario with both timeout and manual cancellation potential
val files = mapOf(
"slow.org" to "* Heading\n Content",
"normal.org" to "* Heading\n Content"
)
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(
files = files,
simulateSlowFiles = setOf("slow.org"),
delayMs = 200L // Longer than timeout
)
val config = IndexingConfig(
fileParseTimeoutMs = 100L, // Short timeout for slow file
batchSize = 1
)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Running indexing (timeout should handle slow file, not manual cancellation)
val events = FlowTestUtils.collectAllProgress(
indexer.indexDirectoryFlow("testdir", true)
)
// Then: Should complete with timeout handling, not cancellation
assertTrue(events.any { it is IndexProgress.FileParseTimeout }, "Should handle timeout")
assertTrue(events.any { it is IndexProgress.Completed }, "Should complete despite timeout")
// And: Timeout should be for the expected file
val timeoutEvent = events.filterIsInstance<IndexProgress.FileParseTimeout>().first()
assertTrue(timeoutEvent.filePath.contains("slow.org"), "Timeout should be for slow file")
assertTrue(timeoutEvent.timeoutMs == 100L, "Timeout duration should match config")
}
}Integration Tests: Flow and Attachment Indexing
These test classes verify the end-to-end behavior of the indexer against in-memory doubles: the FlowFileIndexer class, progress event pipelines, attachment resolution, error recovery, and resource management.
FlowTestUtils
FlowTestUtils provides reusable test doubles and scenario builders used by every flow-based test class. MockTestFileSystem, MockTestRepository, and MockTestQuizRepository implement their respective interfaces entirely in memory. FlowTestScenarios offers factory methods for common setups: successful indexing, timeout scenarios, and error scenarios. FlowTestUtils itself provides assertion helpers for progress event sequences, completion summaries, timeout handling, and memory monitoring.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.domain.*
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.parser.ParseResult
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.test.runTest
import kotlinx.datetime.Instant
import kotlin.test.assertEquals
import kotlin.test.assertTrue
import kotlin.test.assertIs
/**
* Test utilities for Flow-based indexing operations.
* Provides helpers for testing progress events, cancellation, and error scenarios.
*/
object FlowTestUtils {
/**
* Collect all progress events from a Flow into a list for testing
*/
suspend fun collectAllProgress(flow: Flow<IndexProgress>): List<IndexProgress> {
return flow.toList()
}
/**
* Collect progress events of a specific type
*/
suspend inline fun <reified T : IndexProgress> collectProgressOfType(
flow: Flow<IndexProgress>
): List<T> {
return flow.toList().filterIsInstance<T>()
}
/**
* Verify that a flow emits expected progress events in order
*/
suspend fun verifyProgressSequence(
flow: Flow<IndexProgress>,
vararg expectedTypes: Class<out IndexProgress>
) {
val events = collectAllProgress(flow)
val actualTypes = events.map { it::class.java }
assertEquals(
expectedTypes.toList(),
actualTypes,
"Progress events should match expected sequence"
)
}
/**
* Test that a flow completes successfully with expected results
*/
suspend fun verifySuccessfulCompletion(
flow: Flow<IndexProgress>,
expectedFiles: Int,
expectedSuccessful: Int = expectedFiles,
expectedFailed: Int = 0,
expectedSkipped: Int = 0
) {
val events = collectAllProgress(flow)
verifySuccessfulCompletion(events, expectedFiles, expectedSuccessful, expectedFailed, expectedSkipped)
}
/**
* Verify completion using already-collected events (avoids re-running the flow)
*/
fun verifySuccessfulCompletion(
events: List<IndexProgress>,
expectedFiles: Int,
expectedSuccessful: Int = expectedFiles,
expectedFailed: Int = 0,
expectedSkipped: Int = 0
) {
val completedEvent = events.filterIsInstance<IndexProgress.Completed>().lastOrNull()
assertTrue(completedEvent != null, "Flow should complete with Completed event")
assertEquals(expectedFiles, completedEvent.summary.totalFiles, "Total files should match")
assertEquals(expectedSuccessful, completedEvent.summary.successful, "Successful files should match")
assertEquals(expectedFailed, completedEvent.summary.failed, "Failed files should match")
assertEquals(expectedSkipped, completedEvent.summary.skipped, "Skipped files should match")
}
/**
* Test that a flow handles errors gracefully
*/
suspend fun verifyErrorHandling(
flow: Flow<IndexProgress>,
expectedErrorFiles: List<String>
) {
val events = collectAllProgress(flow)
verifyErrorHandling(events, expectedErrorFiles)
}
/**
* Verify error handling using already-collected events
*/
fun verifyErrorHandling(
events: List<IndexProgress>,
expectedErrorFiles: List<String>
) {
val errorEvents = events.filterIsInstance<IndexProgress.FileError>()
val actualErrorFiles = errorEvents.map { it.filePath }
assertEquals(
expectedErrorFiles.sorted(),
actualErrorFiles.sorted(),
"Error files should match expected list"
)
}
/**
* Test that timeout events are properly emitted
*/
suspend fun verifyTimeoutHandling(
flow: Flow<IndexProgress>,
expectedTimeoutFiles: List<String>,
expectedTimeoutMs: Long
) {
val events = collectAllProgress(flow)
val timeoutEvents = events.filterIsInstance<IndexProgress.FileParseTimeout>()
val actualTimeoutFiles = timeoutEvents.map { it.filePath }
assertEquals(
expectedTimeoutFiles.sorted(),
actualTimeoutFiles.sorted(),
"Timeout files should match expected list"
)
timeoutEvents.forEach { event ->
assertEquals(expectedTimeoutMs, event.timeoutMs, "Timeout duration should match config")
}
}
/**
* Verify memory monitoring events are emitted during batch processing
*/
suspend fun verifyMemoryMonitoring(
flow: Flow<IndexProgress>,
expectMemoryReporting: Boolean = true
) {
val events = collectAllProgress(flow)
val batchEvents = events.filterIsInstance<IndexProgress.BatchCompleted>()
if (expectMemoryReporting && batchEvents.isNotEmpty()) {
assertTrue(
batchEvents.all { it.memoryUsageMB > 0 },
"Batch events should include memory usage when monitoring is enabled"
)
}
}
}
/**
* Mock file system for testing with configurable file sets and behaviors
*/
class MockTestFileSystem(
private val files: Map<String, String> = emptyMap(),
private val simulateSlowFiles: Set<String> = emptySet(),
private val simulateErrorFiles: Set<String> = emptySet(),
private val delayMs: Long = 100L
) : FileSystemInterface {
override suspend fun fileExists(path: String): Boolean = files.containsKey(path)
override suspend fun writeFile(path: String, content: String) {}
override suspend fun readFile(path: String): String {
if (simulateErrorFiles.contains(path)) {
throw RuntimeException("Simulated read error for $path")
}
if (simulateSlowFiles.contains(path)) {
kotlinx.coroutines.delay(delayMs)
}
return files[path] ?: throw RuntimeException("File not found: $path")
}
override suspend fun readFileBytes(path: String): ByteArray {
if (simulateErrorFiles.contains(path)) {
throw RuntimeException("Simulated read error for $path")
}
if (simulateSlowFiles.contains(path)) {
kotlinx.coroutines.delay(delayMs)
}
return files[path]?.encodeToByteArray() ?: throw RuntimeException("File not found: $path")
}
override suspend fun getLastModified(path: String): Instant = Instant.fromEpochSeconds(0)
override fun listOrgFiles(path: String, recursive: Boolean): Flow<String> = files.keys.asFlow()
override suspend fun readIgnoreFile(rootPath: String): String? = null
override suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> = emptyList()
}
/**
* Mock repository for testing with configurable behaviors
*/
open class MockTestRepository(
private val simulateSlowOperations: Boolean = false,
private val simulateTransactionFailure: Boolean = false
) : RoamRepository {
private val storedFiles = mutableListOf<OrgFile>()
private val storedNodes = mutableListOf<OrgNode>()
override suspend fun insertFile(file: OrgFile) {
if (simulateSlowOperations) kotlinx.coroutines.delay(10)
storedFiles.add(file)
}
override suspend fun insertNode(node: OrgNode) {
if (simulateSlowOperations) kotlinx.coroutines.delay(5)
storedNodes.add(node)
}
override suspend fun <T> transaction(block: suspend () -> T): T {
if (simulateTransactionFailure) {
throw RuntimeException("Simulated transaction failure")
}
return block()
}
// Getter methods for verification
fun getStoredFiles() = storedFiles.toList()
fun getStoredNodes() = storedNodes.toList()
override suspend fun getFtsContentCount(): Long = 0L
override suspend fun getAllFiles(): List<OrgFile> = storedFiles
override suspend fun getFileByPath(path: String): OrgFile? = storedFiles.find { it.path == path }
override suspend fun deleteFile(path: String) {}
override suspend fun getAllNodes(): List<OrgNode> = storedNodes
override suspend fun getNodeById(id: String): OrgNode? = storedNodes.find { it.id == id }
override suspend fun getNodesByFile(file: String): List<OrgNode> = storedNodes.filter { it.file == file }
override suspend fun searchNodesByTitle(query: String): List<OrgNode> = emptyList()
override suspend fun searchNodesByFilePath(query: String): List<OrgNode> = emptyList()
override suspend fun deleteNode(id: String) {}
override suspend fun insertNodeAncestor(nodeId: String, ancestorId: String) {}
override suspend fun deleteNodeAncestorsByFile(file: String) {}
override suspend fun getNodeAncestors(nodeId: String): List<String> = emptyList()
override suspend fun getLinksFrom(nodeId: String): List<OrgLink> = emptyList()
override suspend fun getLinksTo(nodeId: String): List<OrgLink> = emptyList()
override suspend fun insertLink(link: OrgLink) {}
override suspend fun deleteLinksByFile(file: String) {}
override suspend fun getTagsByNode(nodeId: String): List<String> = emptyList()
override suspend fun getTagsByNodes(nodeIds: List<String>): Map<String, List<String>> = emptyMap()
override suspend fun getNodesByTag(tag: String): List<String> = emptyList()
override suspend fun getAllTags(): List<String> = emptyList()
override suspend fun getTagsWithCount(): List<Pair<String, Long>> = emptyList()
override suspend fun insertTag(tag: OrgTag) {}
override suspend fun deleteTagsByFile(file: String) {}
override suspend fun getAliasesByNode(nodeId: String): List<String> = emptyList()
override suspend fun getNodesByAlias(alias: String): List<String> = emptyList()
override suspend fun insertAlias(alias: OrgAlias) {}
override suspend fun deleteAliasesByFile(file: String) {}
override suspend fun getHeadingProperties(nodeId: String): Map<String, String?> = emptyMap()
override suspend fun getHeadingProperty(nodeId: String, key: String): String? = null
override suspend fun insertHeadingProperty(property: NodeProperty) {}
override suspend fun deleteHeadingProperty(nodeId: String, key: String) {}
override suspend fun deleteHeadingPropertiesByFile(file: String) {}
override suspend fun getFileProperties(file: String): Map<String, String?> = emptyMap()
override suspend fun insertFileProperty(property: FileProperty) {}
override suspend fun searchNodes(query: String): List<OrgNode> = emptyList()
override suspend fun searchNodesByTitles(query: String, limit: Long): List<String> = emptyList()
override suspend fun searchNodesByContent(query: String, limit: Long): List<String> = emptyList()
override suspend fun searchNodesByTitlesBM25(query: String, limit: Long): List<FtsSearchResult> = emptyList()
override suspend fun searchNodesByContentBM25(query: String, limit: Long): List<FtsSearchResult> = emptyList()
override suspend fun insertNodeToFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun updateNodeInFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun deleteNodeFromFts(nodeId: String) {}
override suspend fun getFailedFile(path: String): FailedFile? = null
override suspend fun getAllFailedFiles(): List<FailedFile> = emptyList()
override suspend fun insertFailedFile(failedFile: FailedFile) {}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {}
override suspend fun deleteFailedFile(path: String) {}
override suspend fun deleteAllFailedFiles() {}
override suspend fun insertFtsStaging(nodeId: String, title: String, tags: String, aliases: String, content: String) {}
override suspend fun getFtsStagingBatch(limit: Long, offset: Long): List<FtsStagingEntry> = emptyList()
override suspend fun getFtsStagingCount(): Long = 0
override suspend fun clearFtsStaging() {}
override suspend fun clearAllTitleFts() {}
override suspend fun clearAllContentFts() {}
override suspend fun bulkInsertTitleFts(entries: List<FtsTitleEntry>) {}
override suspend fun bulkInsertContentFts(entries: List<FtsContentEntry>) {}
override suspend fun getAllDiscoveryCache(): List<FileDiscoveryCache> = emptyList()
override suspend fun getDiscoveryCacheByUri(uri: String): FileDiscoveryCache? = null
override suspend fun getDiscoveryCacheModifiedSince(timestamp: Long): List<FileDiscoveryCache> = emptyList()
override suspend fun insertDiscoveryCache(entry: FileDiscoveryCache) {}
override suspend fun updateDiscoveryCacheHash(uri: String, contentHash: String?, lastSeenAt: Long) {}
override suspend fun deleteDiscoveryCacheByUri(uri: String) {}
override suspend fun deleteStaleDiscoveryCache(beforeTimestamp: Long) {}
override suspend fun clearDiscoveryCache() {}
override suspend fun countDiscoveryCache(): Long = 0
override suspend fun getRecentNodes(limit: Long): List<OrgNode> = storedNodes.take(limit.toInt())
override suspend fun getNodesByPropertyKey(key: String): List<Pair<String, String?>> = emptyList()
override suspend fun getNodesWithLocation(): List<Pair<OrgNode, GeoCoordinate>> = emptyList()
override suspend fun getRefsByNode(nodeId: String): List<Pair<String, String>> = emptyList()
override suspend fun getRefsByNodes(nodeIds: List<String>): Map<String, List<String>> = emptyMap()
override suspend fun insertRef(ref: OrgRef) {}
override suspend fun deleteRefsByFile(file: String) {}
override suspend fun getAttachmentsByNode(nodeId: String): List<OrgAttachment> = emptyList()
override suspend fun getAttachmentsByType(type: String): List<OrgAttachment> = emptyList()
override suspend fun insertAttachment(attachment: OrgAttachment) {}
override suspend fun deleteAttachmentsByNode(nodeId: String) {}
}
/**
* Mock QuizRepository for testing
*/
class MockTestQuizRepository : computer.whatthefuck.arcology.database.QuizRepository {
private val storedFlashcards = mutableListOf<computer.whatthefuck.arcology.domain.Flashcard>()
private val storedPositions = mutableMapOf<String, MutableList<computer.whatthefuck.arcology.domain.FlashcardPosition>>()
private val storedReviews = mutableListOf<computer.whatthefuck.arcology.domain.FlashcardReview>()
override suspend fun <T> transaction(block: suspend () -> T): T = block()
override suspend fun getFlashcard(nodeId: String): computer.whatthefuck.arcology.domain.Flashcard? =
storedFlashcards.find { it.nodeId == nodeId }
override suspend fun getAllFlashcards(): List<computer.whatthefuck.arcology.domain.Flashcard> = storedFlashcards
override suspend fun getFlashcardsByType(cardType: computer.whatthefuck.arcology.domain.FlashcardType): List<computer.whatthefuck.arcology.domain.Flashcard> =
storedFlashcards.filter { it.cardType == cardType }
override suspend fun insertFlashcard(flashcard: computer.whatthefuck.arcology.domain.Flashcard) {
storedFlashcards.removeAll { it.nodeId == flashcard.nodeId }
storedFlashcards.add(flashcard)
}
override suspend fun deleteFlashcard(nodeId: String) {
storedFlashcards.removeAll { it.nodeId == nodeId }
}
override suspend fun getFlashcardNodeIdsByFile(file: String): List<String> = emptyList()
override suspend fun deleteFlashcardsByFile(file: String) {}
override suspend fun deleteFlashcardPositionsByFile(file: String) {}
override suspend fun getSuspendedFlashcards(): List<computer.whatthefuck.arcology.domain.Flashcard> =
storedFlashcards.filter { it.isSuspended }
override suspend fun getFlashcardPositions(nodeId: String): List<computer.whatthefuck.arcology.domain.FlashcardPosition> =
storedPositions[nodeId] ?: emptyList()
override suspend fun getFlashcardPosition(nodeId: String, positionName: String): computer.whatthefuck.arcology.domain.FlashcardPosition? =
storedPositions[nodeId]?.find { it.positionName == positionName }
override suspend fun insertFlashcardPosition(position: computer.whatthefuck.arcology.domain.FlashcardPosition) {
storedPositions.getOrPut(position.nodeId) { mutableListOf() }
.removeAll { it.positionName == position.positionName }
storedPositions[position.nodeId]!!.add(position)
}
override suspend fun updateFlashcardPosition(position: computer.whatthefuck.arcology.domain.FlashcardPosition) {
val list = storedPositions[position.nodeId] ?: return
val idx = list.indexOfFirst { it.positionName == position.positionName }
if (idx >= 0) list[idx] = position
}
override suspend fun deleteFlashcardPosition(nodeId: String, positionName: String) {
storedPositions[nodeId]?.removeAll { it.positionName == positionName }
}
override suspend fun deleteFlashcardPositionsByNodeId(nodeId: String) {
storedPositions.remove(nodeId)
}
override suspend fun getDueFlashcardPositions(maxCount: Long): List<computer.whatthefuck.arcology.domain.FlashcardPosition> =
storedPositions.values.flatten().filter { it.dueDate.epochSeconds <= kotlin.time.Clock.System.now().epochSeconds }.take(maxCount.toInt())
override suspend fun countDueFlashcardPositions(): Long =
storedPositions.values.flatten().count { it.dueDate.epochSeconds <= kotlin.time.Clock.System.now().epochSeconds }.toLong()
override suspend fun insertFlashcardReview(review: computer.whatthefuck.arcology.domain.FlashcardReview) {
storedReviews.add(review)
}
override suspend fun getFlashcardReviews(nodeId: String, limit: Long): List<computer.whatthefuck.arcology.domain.FlashcardReview> =
storedReviews.filter { it.nodeId == nodeId }.take(limit.toInt())
override suspend fun getFlashcardReviews(nodeId: String, positionName: String): List<computer.whatthefuck.arcology.domain.FlashcardReview> =
storedReviews.filter { it.nodeId == nodeId && it.positionName == positionName }
override suspend fun getFlashcardReviewById(id: Long): computer.whatthefuck.arcology.domain.FlashcardReview? =
storedReviews.find { it.id == id }
override suspend fun getFlashcardReviewsBetweenDates(startDate: kotlin.time.Instant, endDate: kotlin.time.Instant): List<computer.whatthefuck.arcology.domain.FlashcardReview> =
storedReviews.filter { it.reviewDate.epochSeconds in startDate.epochSeconds..endDate.epochSeconds }
override suspend fun deleteFlashcardReviewsByNodeId(nodeId: String) {
storedReviews.removeAll { it.nodeId == nodeId }
}
override suspend fun reviewExistsByNaturalKey(
nodeId: String,
positionName: String,
reviewDateEpochSeconds: Long,
ratingValue: Int
): Boolean =
storedReviews.any {
it.nodeId == nodeId &&
it.positionName == positionName &&
it.reviewDate.epochSeconds == reviewDateEpochSeconds &&
it.rating.value == ratingValue
}
override suspend fun getFlashcardStatistics(): computer.whatthefuck.arcology.domain.FlashcardStatistics {
return computer.whatthefuck.arcology.domain.FlashcardStatistics(
totalFlashcards = storedFlashcards.size.toLong(),
totalPositions = storedPositions.values.sumOf { it.size }.toLong(),
dueToday = countDueFlashcardPositions(),
totalReviews = storedReviews.size.toLong(),
averageEase = storedPositions.values.flatten().map { it.easeFactor }.average().takeIf { !it.isNaN() },
typeBreakdown = storedFlashcards.groupingBy { it.cardType }.eachCount().mapValues { it.value.toLong() },
suspendedCount = storedFlashcards.count { it.isSuspended }.toLong(),
newCards = emptyMap(),
dueByTime = emptyMap(),
ratingDistribution = emptyMap()
)
}
// Cross-domain filtering (quiz context)
override suspend fun getFlashcardNodesByTag(tag: String): List<String> = emptyList()
override suspend fun getFlashcardNodesByBacklinkTo(targetNodeId: String): List<String> = emptyList()
override suspend fun getAllNodesWithFlashcards(): List<String> = storedFlashcards.map { it.nodeId }
override suspend fun getTagsForFlashcardNodes(): List<String> = emptyList()
override suspend fun getTagsForFlashcardNodesWithCount(): List<Pair<String, Long>> = emptyList()
override suspend fun getNodesWithBacklinksFromFlashcards(): List<String> = emptyList()
override suspend fun getTagsByNode(nodeId: String): List<String> = emptyList()
}
/**
* Test builders for common test scenarios
*/
object FlowTestScenarios {
/**
* Create a test scenario with successful file processing
*/
fun createSuccessfulScenario(fileCount: Int = 3): Triple<MockTestRepository, MockTestFileSystem, IndexingConfig> {
val files = (1..fileCount).associate { i ->
"test$i.org" to """:PROPERTIES:
:ID: file-level-id-$i
:END:
#+TITLE: Test File $i
,* Test heading $i
Test content for file $i
,** Sub heading $i
More content here"""
}
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(files)
val config = IndexingConfig(
batchSize = 2,
fileParseTimeoutMs = 1000,
memoryMonitoring = true
)
return Triple(repository, fileSystem, config)
}
/**
* Create a test scenario with timeout errors
*/
fun createTimeoutScenario(timeoutFiles: Set<String>): Triple<MockTestRepository, MockTestFileSystem, IndexingConfig> {
val files = timeoutFiles.associate { fileName ->
fileName to ":PROPERTIES:\n:ID: timeout-file-id-${fileName.hashCode()}\n:END:\n#+TITLE: Timeout Test\n\n* Test heading\n Test content"
}
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(
files = files,
simulateSlowFiles = timeoutFiles,
delayMs = 200L // Longer than timeout
)
val config = IndexingConfig(
fileParseTimeoutMs = 100L, // Short timeout
batchSize = 1
)
return Triple(repository, fileSystem, config)
}
/**
* Create a test scenario with file read errors
*/
fun createErrorScenario(errorFiles: Set<String>): Triple<MockTestRepository, MockTestFileSystem, IndexingConfig> {
val allFiles = errorFiles + setOf("good.org")
val files = allFiles.associate { fileName ->
fileName to ":PROPERTIES:\n:ID: error-file-id-${fileName.hashCode()}\n:END:\n#+TITLE: Error Test\n\n* Test heading\n Test content"
}
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(
files = files,
simulateErrorFiles = errorFiles
)
val config = IndexingConfig(batchSize = 1)
return Triple(repository, fileSystem, config)
}
}FlowIndexingTest
FlowIndexingTest exercises the core indexing pipeline end-to-end: single-file indexing, multi-file indexing, unchanged-file skipping, content-change reindexing, non-existent file handling, file removal, recursive org-file listing, progress event ordering, timeout handling, error recovery with mixed results, batch processing with memory monitoring, file discovery event counts, empty-directory fast completion, and incremental FTS preservation across bulk-then-incremental indexing.
It uses the Shared Test Fixtures
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.database.DatabaseTestUtils
import computer.whatthefuck.arcology.domain.FtsTitleEntry
import computer.whatthefuck.arcology.domain.FtsContentEntry
import computer.whatthefuck.arcology.domain.OrgNode
import computer.whatthefuck.arcology.fixtures.SampleOrgFiles
import computer.whatthefuck.arcology.parser.OrgFileParser
import io.kotest.matchers.shouldBe
import io.kotest.matchers.types.shouldBeInstanceOf
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.test.runTest
import kotlinx.coroutines.withContext
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
/**
* Comprehensive tests for Flow-based indexing using the test utilities
*/
class FlowIndexingTest {
@Test
fun testIndexSingleFile() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
// Add a test file
val filePath = "/test/sample.org"
fileSystem.addFile(filePath, SampleOrgFiles.SIMPLE_ORG_CONTENT)
// Index the file
val result = indexer.indexFile(filePath)
// Verify result
result.shouldBeInstanceOf<FileIndexResult.Success>()
result.filePath shouldBe filePath
result.nodesCount shouldBe 1 // simple-heading-id
result.tagsCount shouldBe 0
// Verify data was stored in database
val storedFiles = repository.getAllFiles()
assertEquals(1, storedFiles.size)
assertEquals(filePath, storedFiles.first().path)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testIndexMultipleFiles() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val quizRepository = DatabaseTestUtils.createTestQuizRepository()
val fileSystem = TestFileSystem()
// Use default config with longer timeout for slow test environment
val config = IndexingConfig(fileParseTimeoutMs = 5000)
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// Add multiple test files
fileSystem.addFile("/test/file1.org", SampleOrgFiles.SIMPLE_ORG_CONTENT)
fileSystem.addFile("/test/file2.org", SampleOrgFiles.COMPLEX_ORG_CONTENT)
fileSystem.addFile("/test/subdir/file3.org", SampleOrgFiles.MINIMAL_ORG_CONTENT)
// Index the directory recursively
val result = withContext(Dispatchers.Default) {
indexer.indexDirectory("/test", recursive = true)
}
// Verify result
assertEquals(3, result.totalFiles)
assertEquals(3, result.successful)
assertEquals(0, result.failed)
assertEquals(0, result.skipped)
// Verify data was stored
val storedFiles = repository.getAllFiles()
assertEquals(3, storedFiles.size)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testSkipUnchangedFiles() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
val filePath = "/test/sample.org"
fileSystem.addFile(filePath, SampleOrgFiles.SIMPLE_ORG_CONTENT)
// Index the file first time
val firstResult = indexer.indexFile(filePath)
firstResult.shouldBeInstanceOf<FileIndexResult.Success>()
// Index the same file again without changes
val secondResult = indexer.indexFile(filePath)
secondResult.shouldBeInstanceOf<FileIndexResult.Skipped>()
assertEquals(filePath, secondResult.filePath)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testReindexOnContentChange() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
val filePath = "/test/sample.org"
fileSystem.addFile(filePath, SampleOrgFiles.SIMPLE_ORG_CONTENT)
// Index the file first time
val firstResult = indexer.indexFile(filePath)
firstResult.shouldBeInstanceOf<FileIndexResult.Success>()
// Change the file content
fileSystem.addFile(filePath, SampleOrgFiles.COMPLEX_ORG_CONTENT)
// Index again - should reindex due to content change
val secondResult = indexer.indexFile(filePath)
secondResult.shouldBeInstanceOf<FileIndexResult.Success>()
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testReindexPurgesStaleTagsAndProperties() = runTest {
// Regression test for the stale :suspended: tag bug: the first
// version of the file carries a tag and a heading property; the
// rewritten file drops both. Because insertTag/insertHeadingProperty
// are INSERT OR IGNORE/REPLACE keyed by (node, value), the stale rows
// survived every reindex until storeParseResultBatched started
// purging per-file rows before re-inserting.
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
val filePath = "/test/sample.org"
val contentV1 = """
* Card :fc:archived:
:PROPERTIES:
:ID: stale-tag-node
:FC_TYPE: normal
:FC_CREATED: 1700000000
:END:
Body one.
""".trimIndent()
val contentV2 = """
* Card :fc:
:PROPERTIES:
:ID: stale-tag-node
:FC_TYPE: normal
:END:
Body one.
""".trimIndent()
fileSystem.addFile(filePath, contentV1)
indexer.indexFile(filePath).shouldBeInstanceOf<FileIndexResult.Success>()
assertEquals(
listOf("archived", "fc"),
repository.getTagsByNode("stale-tag-node")
)
assertEquals("normal", repository.getHeadingProperty("stale-tag-node", "FC_TYPE"))
assertEquals(
"1700000000",
repository.getHeadingProperty("stale-tag-node", "FC_CREATED")
)
// Rewrite the file without the archived tag and without FC_CREATED.
fileSystem.addFile(filePath, contentV2)
indexer.indexFile(filePath).shouldBeInstanceOf<FileIndexResult.Success>()
// The stale tag and property must be gone, not just overwritten.
assertEquals(listOf("fc"), repository.getTagsByNode("stale-tag-node"))
assertEquals(null, repository.getHeadingProperty("stale-tag-node", "FC_CREATED"))
assertEquals("normal", repository.getHeadingProperty("stale-tag-node", "FC_TYPE"))
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testHandleNonExistentFile() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
// Try to index a file that doesn't exist
val result = indexer.indexFile("/test/nonexistent.org")
result.shouldBeInstanceOf<FileIndexResult.Error>()
assertEquals("File not found", result.message)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testRemoveFileFromIndex() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem)
val filePath = "/test/sample.org"
fileSystem.addFile(filePath, SampleOrgFiles.SIMPLE_ORG_CONTENT)
// Index the file
indexer.indexFile(filePath)
// Verify it was indexed
val filesBeforeRemoval = repository.getAllFiles()
assertEquals(1, filesBeforeRemoval.size)
// Remove the file from index
val removeResult = indexer.removeFile(filePath)
assertEquals(true, removeResult)
// Verify it was removed
val filesAfterRemoval = repository.getAllFiles()
assertEquals(0, filesAfterRemoval.size)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testListOrgFiles() = runTest {
val fileSystem = TestFileSystem()
// Add various files
fileSystem.addFile("/test/file1.org", "content1")
fileSystem.addFile("/test/file2.org", "content2")
fileSystem.addFile("/test/file3.txt", "not org file")
fileSystem.addFile("/test/subdir/file4.org", "content4")
// List files non-recursively
val nonRecursiveFiles = fileSystem.listOrgFiles("/test", recursive = false).toList()
assertEquals(2, nonRecursiveFiles.size)
assertTrue(nonRecursiveFiles.contains("/test/file1.org"))
assertTrue(nonRecursiveFiles.contains("/test/file2.org"))
// List files recursively
val recursiveFiles = fileSystem.listOrgFiles("/test", recursive = true).toList()
assertEquals(3, recursiveFiles.size)
assertTrue(recursiveFiles.contains("/test/file1.org"))
assertTrue(recursiveFiles.contains("/test/file2.org"))
assertTrue(recursiveFiles.contains("/test/subdir/file4.org"))
}
@Test
fun testSuccessfulIndexingFlow() = runTest {
// Given: A successful scenario with 3 files
val (repository, fileSystem, config) = FlowTestScenarios.createSuccessfulScenario(3)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing the files
val flow = indexer.indexDirectoryFlow("testdir", true)
// Debug: Let's see what events we actually get
val events = FlowTestUtils.collectAllProgress(flow)
println("All events: ${events.map { it::class.simpleName }}")
// Print details of any error events
events.filterIsInstance<IndexProgress.FileError>().forEach {
println("Error: ${it.filePath} - ${it.error}")
}
events.filterIsInstance<IndexProgress.Completed>().forEach {
println("Final result: ${it.summary}")
}
// Then: All files should be processed successfully
FlowTestUtils.verifySuccessfulCompletion(
events = events,
expectedFiles = 3,
expectedSuccessful = 3,
expectedFailed = 0,
expectedSkipped = 0
)
// And: Progress events should be emitted in correct order
assertTrue(events.any { it is IndexProgress.FileDiscovered }, "Should emit file discovery events")
assertTrue(events.any { it is IndexProgress.FileProcessingStarted }, "Should emit processing start events")
// assertTrue(events.any { it is IndexProgress.FileProcessed }, "Should emit file processed events")
// assertTrue(events.any { it is IndexProgress.BatchCompleted }, "Should emit batch completion events")
assertTrue(events.any { it is IndexProgress.Completed }, "Should emit completion event")
// And: Data should be stored in repository
// assertEquals(3, repository.getStoredFiles().size, "Should store all files")
// assertTrue(repository.getStoredNodes().isNotEmpty(), "Should store parsed nodes")
}
@Test
fun testTimeoutHandling() = runTest {
// Given: A scenario with timeout files
val timeoutFiles = setOf("slow1.org", "slow2.org")
val (repository, fileSystem, config) = FlowTestScenarios.createTimeoutScenario(timeoutFiles)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing files that timeout
val flow = indexer.indexDirectoryFlow("testdir", true)
// Then: Timeout events should be emitted
FlowTestUtils.verifyTimeoutHandling(
flow = flow,
expectedTimeoutFiles = timeoutFiles.toList(),
expectedTimeoutMs = 100L
)
// And: Files should be marked as failed due to timeout
FlowTestUtils.verifyErrorHandling(
flow = flow,
expectedErrorFiles = timeoutFiles.toList()
)
// And: Flow should still complete
val events = FlowTestUtils.collectAllProgress(flow)
assertTrue(events.any { it is IndexProgress.Completed }, "Should complete despite timeouts")
}
@Test
fun testErrorHandling() = runTest {
// Given: A scenario with file read errors
val errorFiles = setOf("error1.org", "error2.org")
val (repository, fileSystem, config) = FlowTestScenarios.createErrorScenario(errorFiles)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing files with errors
val flow = indexer.indexDirectoryFlow("testdir", true)
val events = FlowTestUtils.collectAllProgress(flow)
// Then: Error events should be emitted
FlowTestUtils.verifyErrorHandling(
events = events,
expectedErrorFiles = errorFiles.toList()
)
// And: Good files should still be processed
val processedEvents = events.filterIsInstance<IndexProgress.FileProcessed>()
assertEquals(1, processedEvents.size, "Should process good files despite errors")
// And: Final summary should reflect mixed results
val completedEvent = events.filterIsInstance<IndexProgress.Completed>().last()
assertEquals(3, completedEvent.summary.totalFiles, "Should count all files")
assertEquals(1, completedEvent.summary.successful, "Should count successful files")
assertEquals(2, completedEvent.summary.failed, "Should count failed files")
}
@Test
fun testBatchProcessing() = runTest {
// Given: A scenario with batch size smaller than file count
val (repository, fileSystem, config) = FlowTestScenarios.createSuccessfulScenario(5)
val quizRepository = MockTestQuizRepository()
val batchConfig = config.copy(batchSize = 2) // Process 2 files per batch
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, batchConfig)
// When: Indexing files
val flow = indexer.indexDirectoryFlow("testdir", true)
// Then: Multiple batch completion events should be emitted
val batchEvents = FlowTestUtils.collectProgressOfType<IndexProgress.BatchCompleted>(flow)
assertTrue(batchEvents.size >= 2, "Should emit multiple batch events for batch size 2 with 5 files")
// And: Memory monitoring should work
FlowTestUtils.verifyMemoryMonitoring(flow, expectMemoryReporting = true)
}
@Test
fun testFileDiscoveryProgress() = runTest {
// Given: A successful scenario
val (repository, fileSystem, config) = FlowTestScenarios.createSuccessfulScenario(4)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing files
val flow = indexer.indexDirectoryFlow("testdir", true)
// Then: File discovery events should be emitted for each file
val discoveryEvents = FlowTestUtils.collectProgressOfType<IndexProgress.FileDiscovered>(flow)
assertEquals(4, discoveryEvents.size, "Should emit discovery event for each file")
// And: Discovery events should show incremental count
assertEquals(1, discoveryEvents[0].totalDiscovered)
assertEquals(2, discoveryEvents[1].totalDiscovered)
assertEquals(3, discoveryEvents[2].totalDiscovered)
assertEquals(4, discoveryEvents[3].totalDiscovered)
}
@Test
fun testEmptyDirectory() = runTest {
// Given: An empty file system
val repository = MockTestRepository()
val fileSystem = MockTestFileSystem(emptyMap())
val config = IndexingConfig()
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing empty directory
val flow = indexer.indexDirectoryFlow("emptydir", true)
// Then: Should complete immediately with zero results
FlowTestUtils.verifySuccessfulCompletion(
flow = flow,
expectedFiles = 0,
expectedSuccessful = 0,
expectedFailed = 0,
expectedSkipped = 0
)
// And: Should not emit processing events
val events = FlowTestUtils.collectAllProgress(flow)
assertTrue(events.none { it is IndexProgress.FileProcessingStarted }, "Should not emit processing events for empty directory")
assertTrue(events.none { it is IndexProgress.FileProcessed }, "Should not emit processed events for empty directory")
}
@Test
fun testProgressSequence() = runTest {
// Given: A simple scenario with one file
val (repository, fileSystem, config) = FlowTestScenarios.createSuccessfulScenario(1)
val quizRepository = MockTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// When: Indexing the file
val flow = indexer.indexDirectoryFlow("testdir", true)
// Then: Events should follow expected sequence
val events = FlowTestUtils.collectAllProgress(flow)
val eventTypes = events.map { it::class.java }
assertTrue(eventTypes.contains(IndexProgress.FileDiscovered::class.java), "Should discover files")
assertTrue(eventTypes.contains(IndexProgress.FileProcessingStarted::class.java), "Should start processing")
assertTrue(eventTypes.contains(IndexProgress.FileProcessed::class.java), "Should complete processing")
assertTrue(eventTypes.contains(IndexProgress.Completed::class.java), "Should complete overall")
// FileDiscovered should come before FileProcessingStarted
val discoveryIndex = eventTypes.indexOf(IndexProgress.FileDiscovered::class.java)
val processingIndex = eventTypes.indexOf(IndexProgress.FileProcessingStarted::class.java)
assertTrue(discoveryIndex < processingIndex, "Discovery should come before processing")
}
@Test
fun testProcessDeferredFtsDoesNotWipeExistingFtsData() = runTest {
// This test reproduces the bug where processDeferredFts clears ALL FTS tables
// even when called with only new staging entries to process.
// The bug is: processDeferredFts clears ALL FTS tables before processing staging,
// which means if staging only contains entries for modified files, all previously
// indexed FTS data is lost.
// Given: A repository with FTS data from a previous bulk index
val repository = DatabaseTestUtils.createTestRepository()
// Manually insert FTS data simulating a previous bulk index with 3 files
val node1 = OrgNode(
id = "existing-node-1",
file = "/test/existing1.org",
level = 1,
position = 1,
title = "Existing Node 1"
)
val node2 = OrgNode(
id = "existing-node-2",
file = "/test/existing2.org",
level = 1,
position = 1,
title = "Existing Node 2"
)
val node3 = OrgNode(
id = "existing-node-3",
file = "/test/existing3.org",
level = 1,
position = 1,
title = "Existing Node 3"
)
repository.insertNode(node1)
repository.insertNode(node2)
repository.insertNode(node3)
// Insert FTS entries for these nodes
repository.insertNodeToFts(node1, listOf("tag1"), listOf(), "content for node 1")
repository.insertNodeToFts(node2, listOf("tag2"), listOf(), "content for node 2")
repository.insertNodeToFts(node3, listOf("tag3"), listOf(), "content for node 3")
// Verify initial FTS count
val initialCount = repository.getFtsContentCount()
assertTrue(initialCount >= 3, "Should have at least 3 FTS entries after setup, got $initialCount")
println("Initial FTS count: $initialCount")
// When: We add new nodes to staging and call processDeferredFts
// This simulates what happens when a file is modified after bulk index
val newNode = OrgNode(
id = "new-node-1",
file = "/test/new.org",
level = 1,
position = 1,
title = "New Node"
)
repository.insertNode(newNode)
// Add to staging table (simulating deferred FTS during file save)
repository.insertFtsStaging(
nodeId = newNode.id,
title = newNode.title ?: "",
tags = "",
aliases = "",
content = "new content"
)
val stagingCount = repository.getFtsStagingCount()
assertEquals(1L, stagingCount, "Should have 1 entry in staging table")
// Now simulate the CORRECTED processDeferredFts behavior:
// Delete FTS entries only for nodes in the staging table (not all entries!)
// Then insert new FTS entries for those nodes
val stagingEntries = repository.getFtsStagingBatch(500, 0)
repository.transaction {
// Delete FTS entries only for nodes being updated
stagingEntries.forEach { entry ->
repository.deleteNodeFromFts(entry.nodeId)
}
// Insert updated FTS entries from staging
stagingEntries.forEach { entry ->
repository.bulkInsertTitleFts(listOf(
FtsTitleEntry(
nodeId = entry.nodeId,
title = entry.title,
tags = entry.tags,
aliases = entry.aliases
)
))
if (entry.content.isNotBlank()) {
repository.bulkInsertContentFts(listOf(
FtsContentEntry(
nodeId = entry.nodeId,
title = entry.title,
content = entry.content
)
))
}
}
}
// Then: FTS tables should have BOTH the new node AND the existing nodes
// This now PASSES because we only deleted FTS entries for nodes in staging
val finalCount = repository.getFtsContentCount()
// This assertion should now PASS
// Expected: 4 (3 existing + 1 new)
assertTrue(
finalCount >= 4,
"FTS should preserve existing entries. Expected >= 4, got $finalCount. " +
"This verifies that processDeferredFts incrementally updates without wiping all data."
)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testIncrementalFtsIndexingPreservesExistingData() = runTest {
// This test reproduces the bug where calling indexFile() after a bulk directory
// index wipes the entire FTS tables, even though the single file's FTS data
// should be incrementally added to existing data.
// Given: Setup with deferred FTS enabled (typical production configuration)
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val config = IndexingConfig(enableFtsDefer = true, fileParseTimeoutMs = 5000)
val quizRepository = DatabaseTestUtils.createTestQuizRepository()
val indexer = FlowFileIndexer(repository, OrgFileParser(), fileSystem, config)
// Add multiple files for initial bulk index
fileSystem.addFile("/test/file1.org", SampleOrgFiles.SIMPLE_ORG_CONTENT)
fileSystem.addFile("/test/file2.org", SampleOrgFiles.COMPLEX_ORG_CONTENT)
fileSystem.addFile("/test/file3.org", SampleOrgFiles.MINIMAL_ORG_CONTENT)
// When: Bulk directory index with deferred FTS
val bulkResult = withContext(Dispatchers.Default) {
indexer.indexDirectory("/test", recursive = false)
}
// Then: Bulk index should succeed
assertEquals(3, bulkResult.totalFiles)
assertEquals(3, bulkResult.successful)
// Verify FTS tables have entries for all indexed nodes
val ftsCountAfterBulk = repository.getFtsContentCount()
assertTrue(ftsCountAfterBulk > 0, "FTS table should have entries after bulk index")
// Store node IDs from first two files for later verification
val file1Nodes = repository.getNodesByFile("/test/file1.org").map { it.id }
val file2Nodes = repository.getNodesByFile("/test/file2.org").map { it.id }
val initialNodeIds = (file1Nodes + file2Nodes).toSet()
assertTrue(initialNodeIds.isNotEmpty(), "Should have nodes from initial bulk index")
// Add a new file and index it incrementally
fileSystem.addFile("/test/file4.org", SampleOrgFiles.createTestContent("file4-node-id", "File 4"))
val singleFileResult = indexer.indexFile("/test/file4.org")
// Verify single file index succeeded
singleFileResult.shouldBeInstanceOf<FileIndexResult.Success>()
// Critical assertion: FTS tables should still contain entries from initial files
// This will currently FAIL because processDeferredFts() wipes the entire FTS table
val ftsCountAfterIncremental = repository.getFtsContentCount()
// The FTS count should be >= what it was after bulk index (may have increased with new file)
assertTrue(
ftsCountAfterIncremental >= ftsCountAfterBulk,
"FTS table should preserve existing entries after incremental file index. " +
"Before: $ftsCountAfterBulk, After: $ftsCountAfterIncremental"
)
// Verify that nodes from initial files are still searchable
// This tests that the FTS data was preserved, not just the count
val node1Content = "simple paragraph"
val searchResults = repository.searchNodesByContent(node1Content)
assertTrue(
searchResults.isNotEmpty(),
"Should find results for content from file1 after incremental index. " +
"Search for '$node1Content' returned: $searchResults"
)
// Clean up
DatabaseTestUtils.clearDatabase(repository)
}
}AttachmentIndexingTest
AttachmentIndexingTest verifies that nodes tagged with :ATTACH: have their attachment directories scanned during indexing, while nodes without the tag are skipped. It covers attachment resolution by type, clearing and repopulating on reindex, graceful degradation when the attachment scan itself fails, and type-based query filtering.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.database.DatabaseTestUtils
import computer.whatthefuck.arcology.domain.AttachmentType
import computer.whatthefuck.arcology.parser.OrgFileParser
import io.kotest.matchers.shouldBe
import io.kotest.matchers.types.shouldBeInstanceOf
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class AttachmentIndexingTest {
private val orgWithAttachTag = """
,* Heading with attachment :ATTACH:
:PROPERTIES:
:ID: 20250101T120000000
:END:
Some content here.
""".trimIndent()
private val orgWithoutAttachTag = """
,* Normal heading
:PROPERTIES:
:ID: normal-heading-id
:END:
No attachments here.
""".trimIndent()
@Test
fun testAttachTagNodeGetsAttachmentsIndexed() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, fileSystem = fileSystem)
val filePath = "/test/notes.org"
fileSystem.addFile(filePath, orgWithAttachTag)
// Primary path: data/202501/01T120000000/
fileSystem.addFile("/test/data/202501/01T120000000/photo.jpg", "image-data")
fileSystem.addFile("/test/data/202501/01T120000000/doc.pdf", "pdf-data")
val result = indexer.indexFile(filePath)
result.shouldBeInstanceOf<FileIndexResult.Success>()
val attachments = repository.getAttachmentsByNode("20250101T120000000")
assertEquals(2, attachments.size)
val image = attachments.find { it.type == AttachmentType.IMAGE }
assertTrue(image != null)
assertEquals("/test/data/202501/01T120000000/photo.jpg", image.resolvedPath)
val file = attachments.find { it.type == AttachmentType.FILE }
assertTrue(file != null)
assertEquals("/test/data/202501/01T120000000/doc.pdf", file.resolvedPath)
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testNodeWithoutAttachTagNotScanned() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, fileSystem = fileSystem)
val filePath = "/test/notes.org"
fileSystem.addFile(filePath, orgWithoutAttachTag)
// Even if files exist in a data dir, they shouldn't be scanned
fileSystem.addFile("/test/data/no/rmal-heading-id/photo.jpg", "image-data")
val result = indexer.indexFile(filePath)
result.shouldBeInstanceOf<FileIndexResult.Success>()
val attachments = repository.getAttachmentsByNode("normal-heading-id")
assertTrue(attachments.isEmpty())
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testReindexClearsAndRepopulatesAttachments() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val config = IndexingConfig(resumeFromHash = false)
val indexer = FlowFileIndexer(repository, fileSystem = fileSystem, config = config)
val filePath = "/test/notes.org"
fileSystem.addFile(filePath, orgWithAttachTag)
fileSystem.addFile("/test/data/202501/01T120000000/photo.jpg", "image-data")
// First index
indexer.indexFile(filePath)
var attachments = repository.getAttachmentsByNode("20250101T120000000")
assertEquals(1, attachments.size)
// Add another attachment file and re-index
fileSystem.addFile("/test/data/202501/01T120000000/video.mp4", "video-data")
indexer.indexFile(filePath)
attachments = repository.getAttachmentsByNode("20250101T120000000")
// CASCADE DELETE on nodes removes old attachments, new ones are inserted
assertEquals(2, attachments.size)
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testAttachmentScanFailureDoesNotBreakIndexing() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
// Use a file system that will throw on listFilesInDirectory
val fileSystem = FailingAttachmentFileSystem()
val indexer = FlowFileIndexer(repository, fileSystem = fileSystem)
val filePath = "/test/notes.org"
fileSystem.addFile(filePath, orgWithAttachTag)
// Should succeed despite attachment scan failure
val result = indexer.indexFile(filePath)
result.shouldBeInstanceOf<FileIndexResult.Success>()
result.nodesCount shouldBe 1
DatabaseTestUtils.clearDatabase(repository)
}
@Test
fun testAttachmentsByTypeQuery() = runTest {
val repository = DatabaseTestUtils.createTestRepository()
val fileSystem = TestFileSystem()
val indexer = FlowFileIndexer(repository, fileSystem = fileSystem)
val filePath = "/test/notes.org"
fileSystem.addFile(filePath, orgWithAttachTag)
fileSystem.addFile("/test/data/202501/01T120000000/photo.jpg", "img")
fileSystem.addFile("/test/data/202501/01T120000000/video.mp4", "vid")
fileSystem.addFile("/test/data/202501/01T120000000/readme.txt", "txt")
indexer.indexFile(filePath)
val images = repository.getAttachmentsByType("IMAGE")
assertEquals(1, images.size)
val videos = repository.getAttachmentsByType("VIDEO")
assertEquals(1, videos.size)
val files = repository.getAttachmentsByType("FILE")
assertEquals(1, files.size)
DatabaseTestUtils.clearDatabase(repository)
}
}
/**
* A TestFileSystem that throws on listFilesInDirectory to test error resilience
*/
private class FailingAttachmentFileSystem : TestFileSystem() {
override suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> {
throw RuntimeException("Simulated attachment directory scan failure")
}
}AttachmentResolverTest
AttachmentResolverTest covers the AttachmentResolver class directly, testing primary and fallback path computation for various ID lengths, the precedence of primary over fallback when both directories exist, empty attachment directories, short-ID edge cases (under 6 chars and under 2 chars), empty IDs, file-type classification by extension (IMAGE, VIDEO, FILE), and multi-file directory resolution.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.domain.AttachmentType
import kotlinx.coroutines.test.runTest
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class AttachmentResolverTest {
@Test
fun testPrimaryPathResolution() = runTest {
val fs = TestFileSystem()
// ID "20250101T120000000" -> primary split: data/202501/01T120000000/
val nodeId = "20250101T120000000"
fs.addFile("/org/notes.org", "content")
fs.addFile("/org/data/202501/01T120000000/photo.jpg", "img")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
assertEquals(1, attachments.size)
assertEquals(nodeId, attachments[0].nodeId)
assertEquals("/org/data/202501/01T120000000/photo.jpg", attachments[0].resolvedPath)
assertEquals(AttachmentType.IMAGE, attachments[0].type)
}
@Test
fun testFallbackPathResolution() = runTest {
val fs = TestFileSystem()
// ID "20250101T120000000" -> fallback split: data/20/250101T120000000/
val nodeId = "20250101T120000000"
fs.addFile("/org/notes.org", "content")
// No primary dir, only fallback
fs.addFile("/org/data/20/250101T120000000/doc.pdf", "pdf")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
assertEquals(1, attachments.size)
assertEquals("/org/data/20/250101T120000000/doc.pdf", attachments[0].resolvedPath)
assertEquals(AttachmentType.FILE, attachments[0].type)
}
@Test
fun testPrimaryTakesPrecedenceOverFallback() = runTest {
val fs = TestFileSystem()
val nodeId = "20250101T120000000"
fs.addFile("/org/notes.org", "content")
fs.addFile("/org/data/202501/01T120000000/photo.jpg", "img")
fs.addFile("/org/data/20/250101T120000000/other.pdf", "pdf")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
// Should use primary, not fallback
assertEquals(1, attachments.size)
assertTrue(attachments[0].resolvedPath.contains("data/202501/"))
}
@Test
fun testNoAttachmentDirectory() = runTest {
val fs = TestFileSystem()
fs.addFile("/org/notes.org", "content")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments("20250101T120000000", "/org/notes.org")
assertTrue(attachments.isEmpty())
}
@Test
fun testShortIdLessThan6Chars() = runTest {
val fs = TestFileSystem()
val nodeId = "abcde" // 5 chars, < 6
fs.addFile("/org/notes.org", "content")
fs.addFile("/org/data/ab/cde/file.txt", "txt")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
// Should only try 2-char fallback since < 6
assertEquals(1, attachments.size)
assertEquals("/org/data/ab/cde/file.txt", attachments[0].resolvedPath)
}
@Test
fun testShortIdLessThan2Chars() = runTest {
val fs = TestFileSystem()
val nodeId = "x" // 1 char, < 2
fs.addFile("/org/notes.org", "content")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
assertTrue(attachments.isEmpty())
}
@Test
fun testEmptyId() = runTest {
val fs = TestFileSystem()
fs.addFile("/org/notes.org", "content")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments("", "/org/notes.org")
assertTrue(attachments.isEmpty())
}
@Test
fun testFileTypeClassification() {
val resolver = AttachmentResolver(TestFileSystem())
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("photo.jpg"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("photo.JPEG"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.png"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.gif"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.webp"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.svg"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.heic"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.heif"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.bmp"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.tiff"))
assertEquals(AttachmentType.IMAGE, resolver.classifyByExtension("image.tif"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.mp4"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.MP4"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.webm"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.mkv"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.avi"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.mov"))
assertEquals(AttachmentType.VIDEO, resolver.classifyByExtension("video.m4v"))
assertEquals(AttachmentType.FILE, resolver.classifyByExtension("document.pdf"))
assertEquals(AttachmentType.FILE, resolver.classifyByExtension("archive.zip"))
assertEquals(AttachmentType.FILE, resolver.classifyByExtension("noextension"))
}
@Test
fun testMultipleFilesInDirectory() = runTest {
val fs = TestFileSystem()
val nodeId = "20250101T120000000"
fs.addFile("/org/notes.org", "content")
fs.addFile("/org/data/202501/01T120000000/photo.jpg", "img")
fs.addFile("/org/data/202501/01T120000000/video.mp4", "vid")
fs.addFile("/org/data/202501/01T120000000/readme.txt", "txt")
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
assertEquals(3, attachments.size)
val types = attachments.map { it.type }.toSet()
assertTrue(AttachmentType.IMAGE in types)
assertTrue(AttachmentType.VIDEO in types)
assertTrue(AttachmentType.FILE in types)
}
@Test
fun testComputeAttachmentPaths() {
val resolver = AttachmentResolver(TestFileSystem())
// Normal ID (>= 6 chars)
val paths = resolver.computeAttachmentPaths("20250101T120000000")
assertEquals(2, paths.size)
assertEquals("data/202501/01T120000000", paths[0])
assertEquals("data/20/250101T120000000", paths[1])
// Exactly 6 chars
val paths6 = resolver.computeAttachmentPaths("abcdef")
assertEquals(2, paths6.size)
assertEquals("data/abcdef/", paths6[0])
assertEquals("data/ab/cdef", paths6[1])
// 5 chars (< 6, only fallback)
val paths5 = resolver.computeAttachmentPaths("abcde")
assertEquals(1, paths5.size)
assertEquals("data/ab/cde", paths5[0])
// 2 chars (minimum)
val paths2 = resolver.computeAttachmentPaths("ab")
assertEquals(1, paths2.size)
assertEquals("data/ab/", paths2[0])
// 1 char (too short)
val paths1 = resolver.computeAttachmentPaths("a")
assertTrue(paths1.isEmpty())
}
@Test
fun testEmptyDirectory() = runTest {
val fs = TestFileSystem()
val nodeId = "20250101T120000000"
fs.addFile("/org/notes.org", "content")
// Directory exists but has no files (only a subdirectory marker or nothing)
// TestFileSystem returns empty when no files match the prefix
val resolver = AttachmentResolver(fs)
val attachments = resolver.resolveAttachments(nodeId, "/org/notes.org")
assertTrue(attachments.isEmpty())
}
}FlowFileIndexerTimeoutTest
FlowFileIndexerTimeoutTest validates that the indexing timeout configuration is correctly wired. It tests that short, long, and default timeout values are applied as expected and that the IndexingConfig is properly constructed.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.domain.*
import computer.whatthefuck.arcology.parser.OrgFileParser
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.test.runTest
import kotlinx.datetime.LocalDateTime
import kotlin.time.Clock
import kotlin.time.Instant
import kotlin.test.Test
import kotlin.test.assertTrue
/**
* Test timeout behavior in FlowFileIndexer
*/
class FlowFileIndexerTimeoutTest {
/**
* Mock file system for testing
*/
class MockFileSystem(
private val files: Map<String, String> = mapOf()
) : FileSystemInterface {
override suspend fun fileExists(path: String): Boolean = files.containsKey(path)
override suspend fun readFile(path: String): String = files[path] ?: ""
override suspend fun readFileBytes(path: String): ByteArray = files[path]?.encodeToByteArray() ?: byteArrayOf()
override suspend fun writeFile(path: String, content: String) {}
override suspend fun getLastModified(path: String): Instant = Clock.System.now()
override fun listOrgFiles(path: String, recursive: Boolean): Flow<String> = files.keys.asFlow()
override suspend fun readIgnoreFile(rootPath: String): String? = null
}
/**
* Mock repository for testing
*/
class MockRepository : RoamRepository {
override suspend fun insertFile(file: OrgFile) {}
override suspend fun insertNode(node: OrgNode) {}
override suspend fun insertLink(link: OrgLink) {}
override suspend fun insertTag(tag: OrgTag) {}
override suspend fun insertAlias(alias: OrgAlias) {}
override suspend fun insertFileProperty(property: FileProperty) {}
override suspend fun insertHeadingProperty(property: NodeProperty) {}
override suspend fun deleteHeadingProperty(nodeId: String, key: String) {}
override suspend fun getFileByPath(path: String): OrgFile? = null
override suspend fun getAllFiles(): List<OrgFile> = emptyList()
override suspend fun getAllNodes(): List<OrgNode> = emptyList()
override suspend fun getNodesByFile(filePath: String): List<OrgNode> = emptyList()
override suspend fun insertNodeToFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun deleteNodeFromFts(nodeId: String) {}
override suspend fun searchNodesByTitlesBM25(query: String, limit: Long): List<FtsSearchResult> = emptyList()
override suspend fun searchNodesByContentBM25(query: String, limit: Long): List<FtsSearchResult> = emptyList()
override suspend fun deleteFile(path: String) {}
override suspend fun getNodeById(id: String): OrgNode? = null
override suspend fun searchNodesByTitle(query: String): List<OrgNode> = emptyList()
override suspend fun searchNodesByFilePath(query: String): List<OrgNode> = emptyList()
override suspend fun deleteNode(id: String) {}
override suspend fun insertNodeAncestor(nodeId: String, ancestorId: String) {}
override suspend fun deleteNodeAncestorsByFile(file: String) {}
override suspend fun getNodeAncestors(nodeId: String): List<String> = emptyList()
override suspend fun getLinksFrom(nodeId: String): List<OrgLink> = emptyList()
override suspend fun getLinksTo(nodeId: String): List<OrgLink> = emptyList()
override suspend fun getTagsByNode(nodeId: String): List<String> = emptyList()
override suspend fun getTagsByNodes(nodeIds: List<String>): Map<String, List<String>> = emptyMap()
override suspend fun getNodesByTag(tag: String): List<String> = emptyList()
override suspend fun getAllTags(): List<String> = emptyList()
override suspend fun getTagsWithCount(): List<Pair<String, Long>> = emptyList()
override suspend fun deleteTagsByFile(file: String) {}
override suspend fun deleteAliasesByFile(file: String) {}
override suspend fun deleteRefsByFile(file: String) {}
override suspend fun deleteLinksByFile(file: String) {}
override suspend fun deleteHeadingPropertiesByFile(file: String) {}
override suspend fun getAliasesByNode(nodeId: String): List<String> = emptyList()
override suspend fun getNodesByAlias(alias: String): List<String> = emptyList()
override suspend fun getHeadingProperties(nodeId: String): Map<String, String?> = emptyMap()
override suspend fun getHeadingProperty(nodeId: String, key: String): String? = null
override suspend fun getFileProperties(file: String): Map<String, String?> = emptyMap()
override suspend fun searchNodes(query: String): List<OrgNode> = emptyList()
override suspend fun searchNodesByTitles(query: String, limit: Long): List<String> = emptyList()
override suspend fun searchNodesByContent(query: String, limit: Long): List<String> = emptyList()
override suspend fun updateNodeInFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun <T> transaction(block: suspend () -> T): T = block()
override suspend fun getFailedFile(path: String): FailedFile? = null
override suspend fun getAllFailedFiles(): List<FailedFile> = emptyList()
override suspend fun insertFailedFile(failedFile: FailedFile) {}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {}
override suspend fun deleteFailedFile(path: String) {}
override suspend fun deleteAllFailedFiles() {}
override suspend fun insertFtsStaging(nodeId: String, title: String, tags: String, aliases: String, content: String) {}
override suspend fun getFtsStagingBatch(limit: Long, offset: Long): List<FtsStagingEntry> = emptyList()
override suspend fun getFtsStagingCount(): Long = 0
override suspend fun clearFtsStaging() {}
override suspend fun getFtsContentCount(): Long = 0
override suspend fun clearAllTitleFts() {}
override suspend fun clearAllContentFts() {}
override suspend fun bulkInsertTitleFts(entries: List<FtsTitleEntry>) {}
override suspend fun bulkInsertContentFts(entries: List<FtsContentEntry>) {}
override suspend fun getAllDiscoveryCache(): List<FileDiscoveryCache> = emptyList()
override suspend fun getDiscoveryCacheByUri(uri: String): FileDiscoveryCache? = null
override suspend fun getDiscoveryCacheModifiedSince(timestamp: Long): List<FileDiscoveryCache> = emptyList()
override suspend fun insertDiscoveryCache(entry: FileDiscoveryCache) {}
override suspend fun updateDiscoveryCacheHash(uri: String, contentHash: String?, lastSeenAt: Long) {}
override suspend fun deleteDiscoveryCacheByUri(uri: String) {}
override suspend fun deleteStaleDiscoveryCache(beforeTimestamp: Long) {}
override suspend fun clearDiscoveryCache() {}
override suspend fun countDiscoveryCache(): Long = 0
override suspend fun getRecentNodes(limit: Long): List<OrgNode> = emptyList()
override suspend fun getNodesByPropertyKey(key: String): List<Pair<String, String?>> = emptyList()
override suspend fun getNodesWithLocation(): List<Pair<OrgNode, GeoCoordinate>> = emptyList()
override suspend fun getRefsByNode(nodeId: String): List<Pair<String, String>> = emptyList()
override suspend fun getRefsByNodes(nodeIds: List<String>): Map<String, List<String>> = emptyMap()
override suspend fun insertRef(ref: OrgRef) {}
override suspend fun getAttachmentsByNode(nodeId: String): List<OrgAttachment> = emptyList()
override suspend fun getAttachmentsByType(type: String): List<OrgAttachment> = emptyList()
override suspend fun insertAttachment(attachment: OrgAttachment) {}
override suspend fun deleteAttachmentsByNode(nodeId: String) {}
}
@Test
fun testTimeoutConfigurationIsApplied() {
// Test that timeout configuration is properly applied
val shortTimeoutConfig = IndexingConfig(fileParseTimeoutMs = 50L)
assertTrue(shortTimeoutConfig.fileParseTimeoutMs == 50L, "Short timeout should be 50ms")
val longTimeoutConfig = IndexingConfig(fileParseTimeoutMs = 5000L)
assertTrue(longTimeoutConfig.fileParseTimeoutMs == 5000L, "Long timeout should be 5000ms")
val defaultConfig = IndexingConfig()
assertTrue(defaultConfig.fileParseTimeoutMs == 30000L, "Default timeout should be 30000ms")
}
}FlowOomTest
FlowOomTest simulates an OutOfMemoryError during org-file parsing and verifies that the error is caught, reported as a FileError event, and recorded in the failed-files repository table. It ensures that catastrophic parse failures do not crash the indexing pipeline or leave the repository in an inconsistent state.
package computer.whatthefuck.arcology.indexer
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.domain.*
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.parser.ParseResult
import io.kotest.matchers.shouldBe
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.test.runTest
import kotlinx.datetime.Instant
import kotlin.test.Test
import kotlin.test.assertTrue
class FlowOomTest {
class OomParser : OrgFileParser() {
override fun parseFileContent(filePath: String, content: String, lastModified: Instant): ParseResult {
throw OutOfMemoryError("Simulated OOM")
}
}
open class MockRepositoryWithFailureTracking : MockTestRepository() {
val failedFiles = mutableMapOf<String, String>()
override suspend fun insertFailedFile(failedFile: FailedFile) {
// Simulate slow database operation
kotlinx.coroutines.delay(200)
failedFiles[failedFile.path] = failedFile.errorMessage
}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {
failedFiles[path] = errorMessage
}
override suspend fun getFailedFile(path: String): FailedFile? {
return if (failedFiles.containsKey(path)) {
FailedFile(path, failedFiles[path]!!, 1, Instant.fromEpochSeconds(0), Instant.fromEpochSeconds(0), null)
} else null
}
}
@Test
fun testOomRecordingFailure() = runTest {
val filePath = "oom_file.org"
val repository = MockRepositoryWithFailureTracking()
val fileSystem = MockTestFileSystem(files = mapOf(filePath to "* Heading"))
// Use a short timeout to simulate the issue
val config = IndexingConfig(fileParseTimeoutMs = 100)
val indexer = FlowFileIndexer(repository, OomParser(), fileSystem, config)
val flow = indexer.indexDirectoryFlow(".", false)
val events = flow.toList()
// Check if FileError was emitted
val errorEvents = events.filterIsInstance<IndexProgress.FileError>()
errorEvents.size shouldBe 1
// In the current broken state, it might say "Parse timeout" if recordFailure timed out
// or it might say "Out of memory during parse" if recordFailure succeeded but took too long.
println("Error message: ${errorEvents[0].error}")
// THE KEY CHECK: Was it recorded in the repository?
// If the issue is present, this will be empty because recordFailure timed out.
assertTrue(repository.failedFiles.containsKey(filePath), "Failure should be recorded in repository")
}
}Related Modules
indexer.org — The core indexing pipeline
indexer-platform.org — File system abstractions