package computer.whatthefuck.arcology.indexer

import kotlinx.coroutines.flow.Flow
import kotlinx.datetime.Instant

/**
 * Abstract interface for file indexing services that provides Flow-based progress reporting.
 * This interface enables dependency injection for testing and makes it easier to integrate
 * with different UI frameworks and environments.
 */
interface FileIndexingService {
    /**
     * Index all org files in the specified directory with real-time progress updates.
     * Returns a Flow of IndexProgress events that can be collected for UI updates.
     *
     * @param path Directory path to index
     * @param recursive Whether to recursively index subdirectories
     * @return Flow of IndexProgress events for real-time progress monitoring
     */
    fun indexDirectoryFlow(path: String, recursive: Boolean = true): Flow<IndexProgress>

    /**
     * Index all org files in the specified directory and return the final result.
     * This is a convenience method that collects the flow and returns only the final result.
     *
     * @param path Directory path to index
     * @param recursive Whether to recursively index subdirectories
     * @return Final IndexResult summary
     */
    suspend fun indexDirectory(path: String, recursive: Boolean = true): IndexResult

    /**
     * Index a single org file.
     *
     * @param filePath Path to the file to index
     * @return Result of the file indexing operation
     */
    suspend fun indexFile(filePath: String): FileIndexResult

    /**
     * Remove a file from the index.
     *
     * @param filePath Path to the file to remove
     * @return True if removal was successful, false otherwise
     */
    suspend fun removeFile(filePath: String): Boolean
}

/**
 * Factory interface for creating FileIndexingService instances with different configurations
 */
interface FileIndexingServiceFactory {
    /**
     * Create a FileIndexingService with the specified configuration
     *
     * @param config Indexing configuration (batch size, timeouts, etc.)
     * @return Configured FileIndexingService instance
     */
    fun createIndexingService(config: IndexingConfig = IndexingConfig()): FileIndexingService

    /**
     * Create a FileIndexingService for testing with mock dependencies
     *
     * @param mockRepository Mock repository for testing
     * @param mockFileSystem Mock file system for testing
     * @param config Test configuration
     * @return Test-configured FileIndexingService instance
     */
    fun createTestIndexingService(
        mockRepository: computer.whatthefuck.arcology.database.RoamRepository,
        mockFileSystem: FileSystemInterface,
        config: IndexingConfig = IndexingConfig()
    ): FileIndexingService
}

/**
 * Interface for file system operations to enable testing
 */
interface FileSystemInterface {
    suspend fun fileExists(path: String): Boolean
    suspend fun readFile(path: String): String
    /**
     * Read file content as bytes. Used for binary files like images.
     *
     * @param path The file path to read
     * @return The file content as a byte array
     */
    suspend fun readFileBytes(path: String): ByteArray {
        // Default implementation reads text and encodes to bytes
        return readFile(path).encodeToByteArray()
    }
    suspend fun writeFile(path: String, content: String)

    /**
     * Append content to a file, creating it if it does not exist.
     *
     * The default implementation reads the current contents, concatenates the
     * new content, and writes the whole file back. This is correct but
     * inefficient for large append-only logs (org-fc's review-history TSV can
     * grow to tens of thousands of rows); platform implementations are
     * encouraged to override with a true append where the underlying API
     * allows it (JVM: [Files.write] with [StandardOpenOption.APPEND];
     * Android SAF: read-merge-write, since DocumentFile has no append mode).
     *
     * Callers that need atomic-ish appends (review-history logging) should
     * treat this as best-effort: there is no cross-platform guarantee that
     * concurrent appends from Emacs and the Android app won't interleave.
     * org-fc itself uses Emacs' [append-to-file] which has the same caveat.
     *
     * @param path The file path to append to
     * @param content The content to append
     */
    suspend fun appendToFile(path: String, content: String) {
        val existing = if (fileExists(path)) readFile(path) else ""
        writeFile(path, existing + content)
    }

    suspend fun getLastModified(path: String): Instant
    fun listOrgFiles(path: String, recursive: Boolean): Flow<String>

    /**
     * List org files with ignore pattern filtering.
     * This version allows filtering files during traversal for better performance.
     *
     * @param path The root path to search
     * @param recursive Whether to search subdirectories
     * @param ignorePatterns Patterns to filter out files/directories
     * @return Flow of file paths that match .org extension and don't match ignore patterns
     */
    fun listOrgFiles(path: String, recursive: Boolean, ignorePatterns: IgnorePatterns): Flow<String> {
        // Default implementation filters after traversal (less efficient but works)
        return listOrgFiles(path, recursive)
    }

    /**
     * Read the .arcologyignore file from the root of the file tree.
     * Returns null if no ignore file exists.
     *
     * @param rootPath The root path of the file tree (used by JVM, ignored by Android which uses treeUri)
     * @return The content of the .arcologyignore file, or null if not found
     */
    suspend fun readIgnoreFile(rootPath: String): String?

    /**
     * List files in a directory relative to a base file's parent directory.
     * Used for attachment scanning.
     *
     * @param baseFilePath Path of a file whose parent directory serves as the base
     * @param relativeDir Relative directory path from the base (e.g. "data/202501/01T120000000")
     * @return List of full file paths (content URIs on Android, filesystem paths on JVM).
     *         Returns empty list if the directory doesn't exist.
     */
    suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> = emptyList()

    /**
     * Copy a database file from source to target location.
     * Platform-specific implementation handles file vs content URI.
     *
     * @param sourcePath The source database file path
     * @param targetPath The target path where the database should be copied
     * @return true if copy succeeded, false otherwise
     */
    suspend fun copyDatabase(sourcePath: String, targetPath: String): Boolean = false
}

/**
 * Result of indexing an entire directory
 */
data class IndexResult(
    val totalFiles: Int,
    val successful: Int,
    val failed: Int,
    val skipped: Int,
    val removed: Int = 0,
    val results: List<FileIndexResult> = emptyList()
)

/**
 * Result of indexing a single file
 */
sealed class FileIndexResult {
    abstract val filePath: String

    data class Success(
        override val filePath: String,
        val nodesCount: Int,
        val linksCount: Int,
        val tagsCount: Int
    ) : FileIndexResult()

    data class Error(
        override val filePath: String,
        val message: String,
        val shouldRecordFailure: Boolean = true,
        val contentHash: String? = null
    ) : FileIndexResult()

    data class Skipped(
        override val filePath: String,
        val reason: String
    ) : FileIndexResult()
}

@file:OptIn(kotlin.time.ExperimentalTime::class)
package computer.whatthefuck.arcology.indexer

import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.domain.*
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.parser.ParseResult
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.withTimeout
import kotlinx.coroutines.TimeoutCancellationException
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.yield
import kotlin.time.ExperimentalTime
import kotlin.time.Instant
import kotlin.time.Clock

/**
 * Flow-based file indexer that provides real-time progress updates
 */
class FlowFileIndexer(
    private val repository: RoamRepository,
    private val parser: OrgFileParser = OrgFileParser(),
    private val fileSystem: FileSystemInterface,
    private val config: IndexingConfig = IndexingConfig(),
    private val plugins: List<IndexerPlugin> = emptyList()
) : FileIndexingService {

}

/**
 ,* Index all org files in the specified directory with real-time progress updates.
 ,* Returns a Flow of IndexProgress events that can be collected for UI updates.
 ,*/
override fun indexDirectoryFlow(path: String, recursive: Boolean): Flow<IndexProgress> = flow {
    try {
        // Phase 0: Load ignore patterns
        <<index-flow-phase0>>

        // Phase 1: Discover all org files
        <<index-flow-phase1>>

        // Phase 2: Remove files from DB that no longer exist on disk
        <<index-flow-phase2>>
        
        // Phase 3: Process files in batches
        <<index-flow-phase3>>

        // Phase 4: FTS indexing (if deferred)
        <<index-flow-phase4>>

        // Phase 5: Export database (if enabled)
        <<index-flow-phase5>>

        // Phase 6: Completion
        <<index-flow-phase6>>

    } catch (e: Exception) {
        emit(IndexProgress.CriticalError(e, "Directory indexing"))
    }
}

val ignorePatterns = loadIgnorePatterns(path)

/**
 ,* Load ignore patterns from the .arcologyignore file at the index root.
 ,* Returns IgnorePatterns.EMPTY if no ignore file exists.
 ,*
 ,* Nested .arcologyignore files in subdirectories are read and merged
 ,* during traversal by the platform file-system walkers.
 ,*/
private suspend fun loadIgnorePatterns(rootPath: String): IgnorePatterns {
    return try {
        val ignoreContent = fileSystem.readIgnoreFile(rootPath)
        if (ignoreContent != null) {
            IgnorePatterns.parse(ignoreContent)
        } else {
            IgnorePatterns.EMPTY
        }
    } catch (e: Exception) {
        IgnorePatterns.EMPTY
    }
}

val discoveredFiles = mutableListOf<String>()
discoverOrgFilesFlow(path, recursive, ignorePatterns)
    .collect { filePath ->
        currentCoroutineContext().ensureActive()
        discoveredFiles.add(filePath)
        emit(IndexProgress.FileDiscovered(filePath, discoveredFiles.size))
        yield()
    }

    /**
     * Discover org files as a Flow with backpressure control
     */
    private fun discoverOrgFilesFlow(path: String, recursive: Boolean, ignorePatterns: IgnorePatterns): Flow<String> {
        return fileSystem.listOrgFiles(path, recursive, ignorePatterns)
    }

val discoveredPathsSet = discoveredFiles.toSet()
val indexedFiles = repository.getAllFiles()
val filesToRemove = indexedFiles.filter { it.path !in discoveredPathsSet }
var totalRemoved = 0

if (filesToRemove.isNotEmpty()) {
    filesToRemove.forEachIndexed { index, file ->
        currentCoroutineContext().ensureActive()

        try {
            val nodes = repository.getNodesByFile(file.path)
            nodes.forEach { node ->
                repository.deleteNodeFromFts(node.id)
            }

            repository.deleteFile(file.path)

            plugins.forEach { plugin ->
                try { plugin.onFileRemoved(file.path) } catch (_: Exception) { }
            }

            emit(IndexProgress.FileRemoved(file.path, index + 1, filesToRemove.size))
            totalRemoved++
        } catch (e: Exception) {
        }

        yield()
    }
}

if (discoveredFiles.isEmpty()) {
    emit(IndexProgress.Completed(IndexResult(0, 0, 0, 0, totalRemoved, emptyList())))
    return@flow
}

var totalProcessed = 0
var totalSuccessful = 0
var totalFailed = 0
var totalSkipped = 0

discoveredFiles.forEachIndexed { index, filePath ->
    currentCoroutineContext().ensureActive()

    emit(IndexProgress.FileProcessingStarted(filePath, index + 1, discoveredFiles.size))

    if (config.memoryMonitoring) {
        val runtime = Runtime.getRuntime()
        val maxMemory = runtime.maxMemory()
        val usedMemory = runtime.totalMemory() - runtime.freeMemory()
        val usedPercent = (usedMemory * 100) / maxMemory
        if (usedPercent > 80) {
            System.gc()
        }
    }

    val result = try {
        withTimeout(config.fileParseTimeoutMs) {
            indexFileWithProgress(filePath, index + 1, discoveredFiles.size)
        }
    } catch (e: TimeoutCancellationException) {
        emit(IndexProgress.FileParseTimeout(
            filePath = filePath,
            timeoutMs = config.fileParseTimeoutMs,
            currentIndex = index + 1,
            totalFiles = discoveredFiles.size
        ))
        FileIndexResult.Error(filePath, "Parse timeout after ${config.fileParseTimeoutMs}ms", shouldRecordFailure = true)
    }

    if (result is FileIndexResult.Error && result.shouldRecordFailure) {
        try {
            recordFailure(filePath, result.message, result.contentHash)
        } catch (e: Exception) {
        }
    }

    when (result) {
        is FileIndexResult.Success -> {
            emit(IndexProgress.FileProcessed(filePath, result, index + 1, discoveredFiles.size))
            totalSuccessful++
        }
        is FileIndexResult.Error -> {
            emit(IndexProgress.FileError(filePath, result.message, index + 1, discoveredFiles.size))
            totalFailed++
        }
        is FileIndexResult.Skipped -> {
            emit(IndexProgress.FileSkipped(filePath, result.reason, index + 1, discoveredFiles.size))
            totalSkipped++
        }
    }

    yield()

    totalProcessed++

    if (totalProcessed % config.batchSize == 0 || totalProcessed == discoveredFiles.size) {
        currentCoroutineContext().ensureActive()

        val batchIndex = (totalProcessed - 1) / config.batchSize + 1
        val memoryUsage = getMemoryUsageMB()
        emit(IndexProgress.BatchCompleted(
            batchIndex = batchIndex,
            batchSize = minOf(config.batchSize, discoveredFiles.size - (batchIndex - 1) * config.batchSize),
            totalProcessed = totalProcessed,
            totalFiles = discoveredFiles.size,
            memoryUsageMB = memoryUsage
        ))

        yield()

        if (config.memoryMonitoring) {
            System.gc()
        }
    }
}

// Phase 2.5: Post-pass plugin hook
// Runs after all files are processed and committed so plugins can
// do one-shot work that needs the full node/card set (e.g. importing
// org-fc's org-fc-reviews.tsv, whose rows foreign-key into
// flashcards written during the per-file pass). Each plugin call
// is wrapped in try/catch so a failing importer can't abort the
// whole indexing run. Not invoked by the single-file indexFile()
// path — see IndexerPlugin.onIndexingComplete.
plugins.forEach { plugin ->
    try {
        plugin.onIndexingComplete(path)
    } catch (_: Exception) { }
}

/**
 ,* Index a single file with integrated progress reporting
 ,*/
private suspend fun indexFileWithProgress(
    filePath: String,
    currentIndex: Int,
    totalFiles: Int,
    forceImmediateFts: Boolean = false
): FileIndexResult {
    var contentHash: String? = null

    try {
        currentCoroutineContext().ensureActive()

        if (!fileSystem.fileExists(filePath)) {
            return FileIndexResult.Error(filePath, "File not found")
        }

        val content = fileSystem.readFile(filePath)

        val lastModified = fileSystem.getLastModified(filePath)

        if (content.isEmpty()) {
            return FileIndexResult.Skipped(filePath, "Empty file (0 bytes)")
        }

        contentHash = computer.whatthefuck.arcology.utils.HashUtils.sha256(content)

        val failedFile = repository.getFailedFile(filePath)
        if (failedFile != null) {
            if (failedFile.fileHash == contentHash) {
                return FileIndexResult.Skipped(filePath, "Previously failed: ${failedFile.errorMessage}")
            } else if (failedFile.fileHash == null) {
                return FileIndexResult.Skipped(filePath, "Previously failed: ${failedFile.errorMessage}")
            } else {
                repository.deleteFailedFile(filePath)
            }
        }

        currentCoroutineContext().ensureActive()

        val existingFile = repository.getFileByPath(filePath)
        val needsReindex = existingFile == null || contentHash != existingFile.hash

        if (!needsReindex && config.resumeFromHash) {
            return FileIndexResult.Skipped(filePath, "File unchanged (hash match)")
        }

        currentCoroutineContext().ensureActive()

        val parseResult = try {
            parser.parseFileContent(filePath, content, lastModified)
        } catch (oom: OutOfMemoryError) {
            System.gc()
            val errorMsg = "Out of memory during parse"
            ParseResult.ParseError(filePath, errorMsg)
        }

        when (parseResult) {
            is ParseResult.Success -> {
                currentCoroutineContext().ensureActive()

                storeParseResultBatched(parseResult, forceImmediateFts)
                return FileIndexResult.Success(
                    filePath = filePath,
                    nodesCount = parseResult.nodes.size,
                    linksCount = parseResult.links.size,
                    tagsCount = parseResult.tags.size
                )
            }
            is ParseResult.ParseError -> {
                return FileIndexResult.Error(filePath, parseResult.error, shouldRecordFailure = true, contentHash = contentHash)
            }
            is ParseResult.FileNotFound -> {
                return FileIndexResult.Error(filePath, "Parser reported file not found", shouldRecordFailure = false)
            }
        }
    } catch (e: Throwable) {
        val errorMessage = e.message ?: e::class.simpleName ?: "Unknown error"
        if (e !is OutOfMemoryError) {
            e.printStackTrace()
        }

        return FileIndexResult.Error(filePath, "Error: $errorMessage", shouldRecordFailure = true, contentHash = contentHash)
    }
}

    /**
     * Record a file parse failure to the database for future skipping
     */
    private suspend fun recordFailure(filePath: String, errorMessage: String, fileHash: String?) {
        try {
            val existingFailure = repository.getFailedFile(filePath)
            if (existingFailure != null) {
                repository.updateFailedFile(filePath, errorMessage, fileHash)
            } else {
                val now = Clock.System.now()
                repository.insertFailedFile(
                    FailedFile(
                        path = filePath,
                        errorMessage = errorMessage,
                        failureCount = 1,
                        firstFailedAt = now,
                        lastFailedAt = now,
                        fileHash = fileHash
                    )
                )
            }
        } catch (e: Exception) {
        }
    }

    /**
     * Store parsed result with transaction-wrapped database operations for atomicity per file
     */
    private suspend fun storeParseResultBatched(result: ParseResult.Success, forceImmediateFts: Boolean = false) {
        repository.transaction {
            val useDefer = config.enableFtsDefer && !forceImmediateFts
            if (!useDefer) {
                val existingNodes = repository.getNodesByFile(result.file.path)
                existingNodes.forEach { node ->
                    repository.deleteNodeFromFts(node.id)
                }
            }

            // Purge the file's previous per-file rows before re-inserting.
            // All the domain tables below use INSERT OR REPLACE/IGNORE keyed by
            // content (node id + tag/alias/key), so rows for tags or properties
            // that were REMOVED from the file would survive every reindex
            // forever. This is what locked in stale :suspended: flashcard
            // tags after a card was unsuspended in the org file: the tags
            // table kept the old row, FlashcardService read it back, and
            // QuizIndexerPlugin re-suspended the card on each pass. Delete
            // first, then insert the fresh parse. nodes themselves are
            // re-keyed by id so insertNode replaces them cleanly; flashcards
            // are reconciled by QuizIndexerPlugin.onFileIndexed.
            repository.deleteTagsByFile(result.file.path)
            repository.deleteAliasesByFile(result.file.path)
            repository.deleteRefsByFile(result.file.path)
            repository.deleteLinksByFile(result.file.path)
            repository.deleteHeadingPropertiesByFile(result.file.path)
            repository.deleteNodeAncestorsByFile(result.file.path)

            repository.insertFile(result.file)

            result.nodes.forEach { node ->
                repository.insertNode(node)

                val nodeTags = result.tags.filter { it.nodeId == node.id }.map { it.tag }
                val nodeAliases = result.aliases.filter { it.nodeId == node.id }.map { it.alias }
                val nodeContent = result.nodeContents[node.id] ?: ""

                if (useDefer) {
                    repository.insertFtsStaging(
                        nodeId = node.id,
                        title = node.title ?: "",
                        tags = nodeTags.joinToString(" "),
                        aliases = nodeAliases.joinToString(" "),
                        content = nodeContent
                    )
                } else {
                    repository.insertNodeToFts(
                        node = node,
                        tags = nodeTags,
                        aliases = nodeAliases,
                        content = nodeContent
                    )
                }
            }

            // Populate node ancestor closure table for property inheritance
            val nodeMap = result.nodes.associateBy { it.id }
            result.nodes.forEach { node ->
                repository.insertNodeAncestor(node.id, node.id)
                var current = node.parentNodeId
                while (current != null) {
                    repository.insertNodeAncestor(node.id, current)
                    current = nodeMap[current]?.parentNodeId
                }
            }

            result.links.forEach { link ->
                repository.insertLink(link)
            }

            result.tags.forEach { tag ->
                repository.insertTag(tag)
            }

            result.aliases.forEach { alias ->
                repository.insertAlias(alias)
            }

            result.refs.forEach { ref ->
                repository.insertRef(ref)
            }

            result.fileProperties.forEach { property ->
                repository.insertFileProperty(property)
            }

            result.nodeProperties.forEach { property ->
                repository.insertHeadingProperty(property)
            }

            // Invoke all registered plugins for domain-specific extraction
            // (Arroyo keywords, flashcards, attachments, etc.)
            plugins.forEach { plugin ->
                try {
                    plugin.onFileIndexed(result)
                } catch (e: Exception) {
                }
            }

            val attachmentResolver = AttachmentResolver(fileSystem)
            val attachTagNodeIds = result.tags
                .filter { it.tag == "ATTACH" }
                .map { it.nodeId }
                .toSet()

            if (attachTagNodeIds.isNotEmpty()) {
                for (nodeId in attachTagNodeIds) {
                    try {
                        val attachments = attachmentResolver.resolveAttachments(nodeId, result.file.path)
                        attachments.forEach { attachment ->
                            repository.insertAttachment(attachment)
                        }
                    } catch (e: Exception) {
                    }
                }
            }
        }
    }

if (config.enableFtsDefer) {
    processDeferredFts().collect { progress ->
        emit(progress)
    }
}

/**
 ,* Process deferred FTS entries from the staging table in batches.
 ,* This method incrementally updates FTS by deleting entries for nodes
 ,* in the staging table, then inserting their updated data.
 ,* Existing FTS entries for nodes NOT in staging are preserved.
 ,*/
private fun processDeferredFts(): Flow<IndexProgress> = flow {
    val totalNodes = repository.getFtsStagingCount()
    if (totalNodes == 0L) {
        return@flow
    }

    emit(IndexProgress.FtsIndexingStarted(totalNodes.toInt()))

    val batchSize = config.ftsBatchSize.toLong()
    val totalBatches = ((totalNodes + batchSize - 1) / batchSize).toInt()
    var processedNodes = 0L

    // Process staging entries in batches
    for (batchIndex in 0 until totalBatches) {
        currentCoroutineContext().ensureActive()

        val offset = batchIndex.toLong() * batchSize
        emit(IndexProgress.FtsBatchStarted(
            batchIndex = batchIndex + 1,
            batchSize = config.ftsBatchSize,
            totalBatches = totalBatches
        ))

        val stagingBatch = repository.getFtsStagingBatch(batchSize, offset)

        if (stagingBatch.isEmpty()) {
            break
        }

        repository.transaction {
            stagingBatch.forEach { entry ->
                repository.deleteNodeFromFts(entry.nodeId)
            }

            val titleEntries = stagingBatch.map { entry ->
                FtsTitleEntry(
                    nodeId = entry.nodeId,
                    title = entry.title,
                    tags = entry.tags,
                    aliases = entry.aliases
                )
            }

            val contentEntries = stagingBatch
                .filter { it.content.isNotBlank() }
                .map { entry ->
                    FtsContentEntry(
                        nodeId = entry.nodeId,
                        title = entry.title,
                        content = entry.content
                    )
                }

            repository.bulkInsertTitleFts(titleEntries)
            repository.bulkInsertContentFts(contentEntries)
        }

        processedNodes += stagingBatch.size
        val memoryUsage = getMemoryUsageMB()

        emit(IndexProgress.FtsBatchCompleted(
            batchIndex = batchIndex + 1,
            processedNodes = processedNodes.toInt(),
            totalNodes = totalNodes.toInt(),
            memoryUsageMB = memoryUsage
        ))

        emit(IndexProgress.FtsProgress(
            processedNodes = processedNodes.toInt(),
            totalNodes = totalNodes.toInt()
        ))

        yield()

        if (config.memoryMonitoring) {
            System.gc()
        }
    }

    repository.clearFtsStaging()

    emit(IndexProgress.FtsCompleted)
}

if (config.exportDatabaseAfterIndexing && config.orgDirectoryPath != null && config.databasePath != null) {
    emit(IndexProgress.DatabaseExportStarted)

    val exportPath = "${config.orgDirectoryPath}/arcology.db"
    val success = fileSystem.copyDatabase(config.databasePath, exportPath)

    emit(IndexProgress.DatabaseExported(success, exportPath))
}

val finalResult = IndexResult(
    totalFiles = discoveredFiles.size,
    successful = totalSuccessful,
    failed = totalFailed,
    skipped = totalSkipped,
    removed = totalRemoved
    // results not accumulated to save memory on Android
)

emit(IndexProgress.Completed(finalResult))

    /**
     * Legacy method for backward compatibility - converts flow to blocking operation
     */
    override suspend fun indexDirectory(path: String, recursive: Boolean): IndexResult {
        var finalResult = IndexResult(0, 0, 0, 0, 0, emptyList())

        indexDirectoryFlow(path, recursive).collect { progress ->
            if (progress is IndexProgress.Completed) {
                finalResult = progress.summary
            }
        }

        return finalResult
    }

    override suspend fun indexFile(filePath: String): FileIndexResult {
        val result = indexFileWithProgress(filePath, 1, 1, forceImmediateFts = true)

        if (config.enableFtsDefer && result is FileIndexResult.Success && result.nodesCount > 0) {
            processDeferredFts()
        }

        return result
    }

    override suspend fun removeFile(filePath: String): Boolean {
        try {
            val nodes = repository.getNodesByFile(filePath)
            nodes.forEach { node ->
                repository.deleteNodeFromFts(node.id)
            }

            repository.deleteFile(filePath)

            plugins.forEach { plugin ->
                try { plugin.onFileRemoved(filePath) } catch (_: Exception) { }
            }

            return true
        } catch (e: Exception) {
            return false
        }
    }

    /**
     * Check if a file needs to be reindexed based on content hash
     */
    private fun shouldReindex(content: String, existingFile: OrgFile?): Boolean {
        if (existingFile == null) {
            return true // New file, needs indexing
        }

        // Calculate content hash and compare
        val newHash = computer.whatthefuck.arcology.utils.HashUtils.sha256(content)
        return newHash != existingFile.hash
    }

    /**
     * Get current memory usage in MB for monitoring
     */
    private fun getMemoryUsageMB(): Long {
        val runtime = Runtime.getRuntime()
        val usedMemory = runtime.totalMemory() - runtime.freeMemory()
        return usedMemory / (1024 * 1024)
    }

package computer.whatthefuck.indexer

import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.subcommands
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.arguments.optional
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.options.flag
import com.github.ajalt.clikt.parameters.types.path
import kotlin.io.path.pathString
import computer.whatthefuck.arcology.database.RoamRepositoryImpl
import computer.whatthefuck.arcology.database.DatabaseFactory
import computer.whatthefuck.arcology.indexer.FlowFileIndexer
import computer.whatthefuck.arcology.indexer.IndexProgress
import computer.whatthefuck.arcology.indexer.IndexingConfig
import computer.whatthefuck.arcology.indexer.JvmFileSystem
import computer.whatthefuck.arcology.indexer.createIndexingService
import computer.whatthefuck.arcology.search.SearchService
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.launch
import kotlinx.coroutines.Job
import java.io.File
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.parser.ParseResult
import kotlin.time.measureTime
import kotlinx.coroutines.CancellationException
import xyz.lepisma.orgmode.lexer.OrgLexer
import xyz.lepisma.orgmode.lexer.Token
import xyz.lepisma.orgmode.core.TimingCollector

class IndexCommand : CliktCommand(name = "index", help = "Index org files in a directory") {
    private val directory by argument(help = "Directory to index").path(canBeFile = false)
    private val recursive by option("--recursive", "-r", help = "Index recursively").default("true")
    private val dbPath by option("--db", help = "Database path").default("arcology.db")
    private val batchSize by option("--batch-size", "-b", help = "Files to process per batch").int().default(50)
    private val enableMemoryMonitoring by option("--memory-monitoring", help = "Enable memory usage monitoring").flag(default = true)

    override fun run() = runBlocking {
        // Handle tilde expansion manually
        val expandedPath = if (directory.pathString.startsWith("~")) {
            val homeDir = System.getProperty("user.home")
            if (directory.pathString == "~") {
                homeDir
            } else if (directory.pathString.startsWith("~/")) {
                homeDir + directory.pathString.substring(1)
            } else {
                directory.pathString // ~user expansion not supported for simplicity
            }
        } else {
            directory.pathString
        }

        // Database is written to the indexed directory by default
        val databasePath = if (dbPath == "arcology.db") {
            java.io.File(expandedPath, "arcology.db").absolutePath
        } else {
            dbPath
        }

        echo("Indexing org files in: $expandedPath")
        echo("Database: $databasePath")
        echo("Recursive: $recursive")

        val config = IndexingConfig(
            batchSize = batchSize,
            memoryMonitoring = enableMemoryMonitoring,
            exportDatabaseAfterIndexing = true,
            orgDirectoryPath = expandedPath,
            databasePath = databasePath
        )
        val indexer = createIndexingService(databasePath, config, expandedPath)

        val reporter = computer.whatthefuck.arcology.cli.CliProgressReporter()
        var totalFiles = 0

        try {
            // Set up signal handling for graceful cancellation
            var indexingJob: Job? = null

            Runtime.getRuntime().addShutdownHook(Thread {
                                                     println("\nReceived shutdown signal (Ctrl+C)...")
                                                     println("Canceling indexing operation gracefully...")
                                                     runBlocking {
                                                         indexingJob?.cancelAndJoin()
                                                     }
                                                     reporter.finish()
                                                     println("Indexing cancelled. Database may be in partial state.")
                                                 })

            coroutineScope {
                indexingJob = launch {
                    indexer.indexDirectoryFlow(expandedPath, recursive.toBoolean()).collect { progress ->
                        when (progress) {
                            is IndexProgress.FileDiscovered -> {
                                totalFiles = progress.totalDiscovered
                                reporter.tickDiscovery(progress.totalDiscovered)
                            }

                            is IndexProgress.FileProcessingStarted -> {
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, progress.filePath)
                            }

                            is IndexProgress.FileProcessed -> {
                                reporter.recordOk()
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, "✓ ${progress.filePath}")
                            }

                            is IndexProgress.FileError -> {
                                reporter.recordFailed()
                                reporter.fail("✗ ${progress.filePath}: ${progress.error}")
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, "error")
                            }

                            is IndexProgress.FileSkipped -> {
                                reporter.recordSkipped()
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, "⊘ ${progress.filePath}")
                            }

                            is IndexProgress.FileRemoved -> {
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, "🗑 removed ${progress.filePath}")
                            }

                            is IndexProgress.FileParseTimeout -> {
                                reporter.recordFailed()
                                reporter.fail("⏱ timeout ${progress.filePath} (${progress.timeoutMs}ms)")
                                reporter.tickWork(progress.currentIndex, progress.totalFiles, "timeout")
                            }

                            is IndexProgress.BatchCompleted -> {
                                val memInfo = if (config.memoryMonitoring) " (Mem: ${progress.memoryUsageMB}MB)" else ""
                                reporter.tickWork(progress.totalProcessed, progress.totalFiles, "batch ${progress.batchIndex}$memInfo")
                            }

                            is IndexProgress.FtsIndexingStarted -> {
                                reporter.finish()
                                echo("Starting FTS indexing for ${progress.totalNodes} nodes...")
                                reporter.startWork("FTS", progress.totalNodes)
                            }

                            is IndexProgress.FtsProgress -> {
                                reporter.tickWork(progress.processedNodes, progress.totalNodes, "fts")
                            }

                            is IndexProgress.FtsBatchStarted -> {
                                reporter.tickWork(progress.batchIndex, progress.totalBatches, "fts batch ${progress.batchIndex}/${progress.totalBatches}")
                            }

                            is IndexProgress.FtsBatchCompleted -> {
                                val memInfo = if (config.memoryMonitoring) " (Mem: ${progress.memoryUsageMB}MB)" else ""
                                reporter.tickWork(progress.processedNodes, progress.totalNodes, "fts batch ${progress.batchIndex}$memInfo")
                            }

                            is IndexProgress.FtsCompleted -> {
                                reporter.finish()
                                echo("FTS indexing completed.")
                            }

                            is IndexProgress.DatabaseExportStarted -> {
                                reporter.finish()
                                echo("Exporting database...")
                            }

                            is IndexProgress.DatabaseExported -> {
                                if (progress.success) {
                                    echo("Database exported to ${progress.targetPath}")
                                } else {
                                    echo("Database export failed", err = true)
                                }
                            }

                            is IndexProgress.Completed -> {
                                reporter.finish()
                                echo("\nIndexing completed:")
                                echo("  Total files: ${progress.summary.totalFiles}")
                                echo("  Successful: ${progress.summary.successful}")
                                echo("  Failed: ${progress.summary.failed}")
                                echo("  Skipped: ${progress.summary.skipped}")
                                if (progress.summary.removed > 0) {
                                    echo("  Removed: ${progress.summary.removed}")
                                }

                                if (progress.summary.failed > 0) {
                                    echo("\nFailed files:")
                                    progress.summary.results.filterIsInstance<computer.whatthefuck.arcology.indexer.FileIndexResult.Error>()
                                        .forEach { error ->
                                            echo("  ${error.filePath}: ${error.message}")
                                        }
                                }
                            }

                            is IndexProgress.CriticalError -> {
                                reporter.finish()
                                val errorMessage = progress.error.message ?: progress.error.javaClass.simpleName
                                echo("Critical error during indexing: $errorMessage", err = true)
                                throw progress.error
                            }
                        }
                    }
                }
            }
        } catch (e: CancellationException) {
            reporter.finish()
            echo("Indexing was cancelled by user.", err = true)
        } catch (e: Exception) {
            reporter.finish()
            val errorMessage = e.message ?: e.javaClass.simpleName
            echo("Error during indexing: $errorMessage", err = true)
        }
    }
}

class SearchCommand : CliktCommand(name = "search", help = "Search indexed org files") {
    private val query by argument(help = "Search query")
    private val dbPath by option("--db", help = "Database path").default("arcology.db")
    private val limit by option("--limit", "-l", help = "Maximum results").int().default(20)
    private val mode by option("--mode", "-m", help = "Search mode: primary, content, combined, simple")
        .default("combined")

    override fun run() = runBlocking {
        if (!File(dbPath).exists()) {
            echo("Database not found: $dbPath", err = true)
            echo("Run 'arcology index <directory>' first to create index.")
            return@runBlocking
        }

        val database = DatabaseFactory.createDatabase(dbPath)
        val repository = RoamRepositoryImpl(database)
        val searchService = SearchService(repository)

        echo("Searching for: '$query'")
        echo("Mode: $mode")
        echo("Limit: $limit")
        echo()

        try {
            when (mode) {
                "primary" -> {
                    val results = searchService.searchPrimary(query, limit)
                    echo("Found ${results.size} results in titles/metadata:")
                    results.forEach { result ->
                        echo("  [${String.format("%.3f", result.rank)}] ${result.node.title} (${result.node.file})")
                    }
                }
                "content" -> {
                    val results = searchService.searchContent(query, limit = limit)
                    echo("Found ${results.size} results in content:")
                    results.forEach { result ->
                        echo("  [${String.format("%.3f", result.rank)}] ${result.node.title} (${result.node.file})")
                    }
                }
                "combined" -> {
                    val results = searchService.searchCombined(query, totalLimit = limit)
                    echo("Found ${results.size} results (combined search):")
                    results.forEach { result ->
                        echo("  [${String.format("%.3f", result.rank)}] ${result.node.title} (${result.node.file}) [${result.source}]")
                    }
                }
                "simple" -> {
                    val results = searchService.searchSimple(query, limit = limit)
                    echo("Found ${results.size} results (simple search):")
                    results.forEach { node ->
                        echo("  ${node.title} (${node.file})")
                    }
                }
                else -> {
                    echo("Invalid search mode: $mode", err = true)
                    echo("Valid modes: primary, content, combined, simple")
                }
            }
        } catch (e: Exception) {
            val errorMessage = e.message ?: e.javaClass.simpleName
            echo("Error during search: $errorMessage", err = true)
        }
    }
}

class StatsCommand : CliktCommand(name = "stats", help = "Show database statistics") {
    private val dbPath by option("--db", help = "Database path").default("arcology.db")

    override fun run() = runBlocking {
        if (!File(dbPath).exists()) {
            echo("Database not found: $dbPath", err = true)
            return@runBlocking
        }

        val database = DatabaseFactory.createDatabase(dbPath)
        val repository = RoamRepositoryImpl(database)

        try {
            val files = repository.getAllFiles()
            val nodes = repository.getAllNodes()

            echo("Database Statistics:")
            echo("  Files indexed: ${files.size}")
            echo("  Nodes (headings): ${nodes.size}")

            if (files.isNotEmpty()) {
                echo("\nRecent files:")
                files.sortedByDescending { it.modificationTime }
                    .take(5)
                    .forEach { file ->
                        echo("  ${file.title ?: file.path} (${file.modificationTime})")
                    }
            }
        } catch (e: Exception) {
            val errorMessage = e.message ?: e.javaClass.simpleName
            echo("Error reading database: $errorMessage", err = true)
        }
    }
}

/**
 * CLI command for single-file parser profiling and debugging.
 *
 * Useful for diagnosing parse failures or performance issues on individual files:
 *
 *   make cli args="parse-file path/to/file.org --skip-fts --profile"
 *     Parses the file and prints a timing table showing Total + Exclusive time
 *     for all major parsers (parseDocument, parseSection, parseChunk, parseInlineElems,
 *     parseHeading, block parsers, and combinator overhead from oneOf/zeroOrMore).
 *     Exclusive time subtracts child parser time, revealing where time is actually spent.
 *
 *   make cli args="parse-file path/to/file.org --skip-fts --dump-tokens"
 *     On parse failure, dumps the full token stream for comparison against parser
 *     expectations. Useful for finding parser/lexer mismatches.
 *
 * The underlying timing infrastructure lives in ParserCore.kt's TimingCollector and
 * the .timed() combinator. All major parsers in OrgBlock.kt, OrgSection.kt,
 * OrgChunk.kt, OrgInlineElem.kt, and OrgDocument.kt are already instrumented.
 * TimingCollector.enabled must be set to true before parsing and false after.
 */
class ParseFileCommand : CliktCommand(name = "parse-file", help = "Parse and profile a single org file") {
    private val filePath by argument(help = "Path to org file to parse").path(mustExist = true, canBeDir = false)
    private val dbPath by option("--db", help = "Database path (optional for FTS testing)").default("arcology.db")
    private val skipFts by option("--skip-fts", help = "Skip FTS indexing for performance testing").flag(default = false)
    private val verbose by option("--verbose", "-v", help = "Show detailed parsing information").flag(default = false)
    private val memoryProfile by option("--memory", "-m", help = "Show memory usage before/after").flag(default = false)
    private val dumpTokens by option("--dump-tokens", help = "Dump all tokenized output on parse failure").flag(default = false)
    private val profile by option("--profile", "-p", help = "Show parser timing profile").flag(default = false)

    override fun run() = runBlocking {
        // Handle tilde expansion manually
        val expandedPath = if (filePath.pathString.startsWith("~")) {
            val homeDir = System.getProperty("user.home")
            if (filePath.pathString == "~") {
                homeDir
            } else if (filePath.pathString.startsWith("~/")) {
                homeDir + filePath.pathString.substring(1)
            } else {
                filePath.pathString
            }
        } else {
            filePath.pathString
        }

        echo("Parsing file: $expandedPath")
        echo("Skip FTS: $skipFts")
        echo("Verbose: $verbose")
        if (memoryProfile) {
            echo("Memory profiling enabled")
        }
        echo()

        val file = File(expandedPath)
        if (!file.exists()) {
            echo("File not found: $expandedPath", err = true)
            return@runBlocking
        }

        val runtime = Runtime.getRuntime()

        fun getMemoryUsageMB(): Long {
            val usedMemory = runtime.totalMemory() - runtime.freeMemory()
            return usedMemory / (1024 * 1024)
        }

        if (memoryProfile) {
            System.gc()
            val initialMemory = getMemoryUsageMB()
            echo("Initial memory usage: ${initialMemory}MB")
        }

        val fileSystem = JvmFileSystem()
        val parser = OrgFileParser()

        try {
            echo("Reading file content...")
            val readTime = measureTime {
                val content = fileSystem.readFile(expandedPath)
                echo("File size: ${content.length} characters")
                echo("File size: ${content.lines().size} lines")

                if (verbose) {
                    echo("First 200 characters:")
                    echo("---")
                    echo(content.take(200))
                    if (content.length > 200) echo("...")
                    echo("---")
                }
            }
            echo("File read time: $readTime")

            if (memoryProfile) {
                val afterReadMemory = getMemoryUsageMB()
                echo("Memory after file read: ${afterReadMemory}MB")
            }

            echo()
            echo("Parsing file structure...")

            var parseResult: ParseResult? = null
            val parseTime = measureTime {
                val content = fileSystem.readFile(expandedPath)
                val lastModified = fileSystem.getLastModified(expandedPath)
                if (profile) {
                    TimingCollector.enabled = true
                    TimingCollector.reset()
                }
                parseResult = parser.parseFileContent(expandedPath, content, lastModified)
                if (profile) {
                    TimingCollector.enabled = false
                }
            }

            echo("Parse time: $parseTime")

            if (profile) {
                dumpProfile()
            }

            if (memoryProfile) {
                val afterParseMemory = getMemoryUsageMB()
                echo("Memory after parsing: ${afterParseMemory}MB")
            }

            when (val result = parseResult) {
                is ParseResult.Success -> {
                    echo()
                    echo("Parse Results:")
                    echo("  File title: ${result.file.title ?: "No title"}")
                    echo("  Nodes (headings): ${result.nodes.size}")
                    echo("  Links: ${result.links.size}")
                    echo("  Tags: ${result.tags.size}")
                    echo("  Aliases: ${result.aliases.size}")
                    echo("  File properties: ${result.fileProperties.size}")
                    echo("  Node properties: ${result.nodeProperties.size}")

                    if (verbose && result.nodes.isNotEmpty()) {
                        echo()
                        echo("Node details:")
                        result.nodes.take(10).forEach { node ->
                            echo("  - ID: ${node.id}")
                            echo("    Title: ${node.title}")
                            echo("    Level: ${node.level}")
                            echo("    Todo: ${node.todo ?: "none"}")
                        }
                        if (result.nodes.size > 10) {
                            echo("  ... and ${result.nodes.size - 10} more nodes")
                        }
                    }

                    if (!skipFts && File(dbPath).exists()) {
                        echo()
                        echo("Testing FTS indexing...")

                        val database = DatabaseFactory.createDatabase(dbPath)
                        val repository = RoamRepositoryImpl(database)

                        val ftsTime = measureTime {
                            result.nodes.forEach { node ->
                                val nodeTags = result.tags.filter { it.nodeId == node.id }.map { it.tag }
                                val nodeAliases = result.aliases.filter { it.nodeId == node.id }.map { it.alias }

                                repository.insertNodeToFts(
                                    node = node,
                                    tags = nodeTags,
                                    aliases = nodeAliases,
                                    content = ""
                                )
                            }
                        }

                        echo("FTS indexing time: $ftsTime")

                        if (memoryProfile) {
                            val afterFtsMemory = getMemoryUsageMB()
                            echo("Memory after FTS indexing: ${afterFtsMemory}MB")
                        }
                    }
                }

                is ParseResult.ParseError -> {
                    echo("Parse Error: ${result.error}", err = true)
                    if (dumpTokens) {
                        dumpTokenDump(File(expandedPath))
                    }
                }

                is ParseResult.FileNotFound -> {
                    echo("File not found error from parser", err = true)
                }

                null -> {
                    echo("No parse result returned", err = true)
                }
            }

            if (memoryProfile) {
                System.gc()
                Thread.sleep(100) // Give GC time to work
                val finalMemory = getMemoryUsageMB()
                echo()
                echo("Final memory usage: ${finalMemory}MB")
            }

        } catch (e: Exception) {
            val errorMessage = e.message ?: e.javaClass.simpleName
            echo("Error during parsing: $errorMessage", err = true)
            if (verbose) {
                echo("Stack trace:")
                e.printStackTrace()
            }
        }
    }

    private fun dumpTokenDump(file: File) {
        val content = file.readText()
        val tokens = OrgLexer(content).tokenize()
        echo()
        echo("=== Token dump (${tokens.size} tokens) ===")
        tokens.forEachIndexed { i, tok ->
            val text = tok.text.take(40).replace("\n", "\\n")
            val cls = tok.javaClass.simpleName
            echo("[%4d] %s [%s]".format(i, text, cls))
        }
    }

    private fun dumpProfile() {
        val entries = TimingCollector.dump()
        if (entries.isEmpty()) return
        echo()
        echo("=== Parser Profile ===")
        echo("%-30s %10s %10s  %8s  %10s".format("Name", "Total", "Exclusive", "Calls", "Avg/excl"))
        echo("-".repeat(82))
        for (e in entries) {
            echo("%-30s %8.2fms %8.2fms  %6d  %9.2f\u00B5s".format(
                e.name, e.totalMs, e.exclusiveMs, e.callCount, e.avgExclusiveMicros
            ))
        }
    }
}

package computer.whatthefuck.arcology.indexer

/**
 * Progress events emitted during the indexing process.
 * These events provide real-time updates on indexing status and can be used
 * for progress bars, status displays, and error reporting.
 */
sealed class IndexProgress {

    /**
     * Emitted when a new org file is discovered during directory scanning
     */
    data class FileDiscovered(
        val filePath: String,
        val totalDiscovered: Int
    ) : IndexProgress()

    /**
     * Emitted when a file starts being processed
     */
    data class FileProcessingStarted(
        val filePath: String,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when a file has been successfully processed
     */
    data class FileProcessed(
        val filePath: String,
        val result: FileIndexResult.Success,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when a file processing fails
     */
    data class FileError(
        val filePath: String,
        val error: String,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when a file is skipped (e.g., no changes detected)
     */
    data class FileSkipped(
        val filePath: String,
        val reason: String,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when an indexed file is removed because it no longer exists on disk
     */
    data class FileRemoved(
        val filePath: String,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when a file parsing times out
     */
    data class FileParseTimeout(
        val filePath: String,
        val timeoutMs: Long,
        val currentIndex: Int,
        val totalFiles: Int
    ) : IndexProgress()

    /**
     * Emitted when a batch of files has been processed and committed to database
     */
    data class BatchCompleted(
        val batchIndex: Int,
        val batchSize: Int,
        val totalProcessed: Int,
        val totalFiles: Int,
        val memoryUsageMB: Long = 0
    ) : IndexProgress()

    /**
     * Emitted when FTS indexing starts (separate from main indexing)
     */
    data class FtsIndexingStarted(
        val totalNodes: Int
    ) : IndexProgress()

    /**
     * Emitted during FTS indexing progress
     */
    data class FtsProgress(
        val processedNodes: Int,
        val totalNodes: Int
    ) : IndexProgress()

    /**
     * Emitted when FTS indexing completes
     */
    data object FtsCompleted : IndexProgress()

    /**
     * Emitted when an FTS batch processing starts (for deferred FTS)
     */
    data class FtsBatchStarted(
        val batchIndex: Int,
        val batchSize: Int,
        val totalBatches: Int
    ) : IndexProgress()

    /**
     * Emitted when an FTS batch processing completes (for deferred FTS)
     */
    data class FtsBatchCompleted(
        val batchIndex: Int,
        val processedNodes: Int,
        val totalNodes: Int,
        val memoryUsageMB: Long = 0
    ) : IndexProgress()

    /**
     * Emitted when database export starts (Phase 5)
     */
    data object DatabaseExportStarted : IndexProgress()

    /**
     * Emitted when database export completes (Phase 5)
     */
    data class DatabaseExported(
        val success: Boolean,
        val targetPath: String
    ) : IndexProgress()

    /**
     * Emitted when the entire indexing operation completes
     */
    data class Completed(
        val summary: IndexResult
    ) : IndexProgress()

    /**
     * Emitted when a critical error occurs that stops the entire operation
     */
    data class CriticalError(
        val error: Throwable,
        val context: String
    ) : IndexProgress()
}

/**
 * Configuration for the flow-based indexing process
 */
data class IndexingConfig(
    val batchSize: Int = 50,
    val maxConcurrency: Int = 4,
    val enableFtsDefer: Boolean = true,
    val memoryMonitoring: Boolean = true,
    val resumeFromHash: Boolean = true,
    val fileParseTimeoutMs: Long = 30000,
    val ftsBatchSize: Int = 500,
    val exportDatabaseAfterIndexing: Boolean = false,
    val orgDirectoryPath: String? = null,
    val databasePath: String? = null
)

package computer.whatthefuck.arcology.indexer

// Glob pattern matching for .arcologyignore files with nested-scope support.
//
// Each IgnorePatterns instance carries a list of scopes. A scope is a pair of
// (baseDir, compiledPatterns) where baseDir is the path of the directory that
// contained the ignore file, relative to the index root (empty string for the
// root ignore file). A candidate path is tested against each scope whose
// baseDir is a prefix of the path; the baseDir prefix is stripped and the
// remainder is matched anchored against that scope's patterns.
//
// Supported glob syntax (anchored to the ignore file's directory):
// - * matches any sequence of characters except /
// - ** matches any sequence of characters including /
// - ? matches any single character except /
// - Lines starting with # are comments
// - Empty lines are ignored

/** Sentinel value so IgnorePatterns' private constructor has a distinct JVM signature. */
private object IgnorePatternsMarker

class IgnorePatterns private constructor(
    private val scopes: List<Scope>,
    @Suppress("UNUSED_PARAMETER") marker: IgnorePatternsMarker
) {

    private data class Scope(val baseDir: String, val patterns: List<Regex>)

    /**
     * Construct an IgnorePatterns rooted at the index root (empty baseDir)
     * from a list of raw pattern lines. Maintained for backwards-compatible
     * call sites that only need a single root-scoped ignore file.
     */
    constructor(patterns: List<String>) : this(listOf(Scope("", patterns.compile())), IgnorePatternsMarker)

    /**
     * Check if a path should be ignored based on any applicable scope.
     *
     * @param path The path to check, relative to the index root
     *             (e.g., "archive/old.org", "notes.org")
     * @return true if the path matches any ignore pattern in any scope whose
     *         baseDir is a prefix of the path
     */
    fun shouldIgnore(path: String): Boolean {
        if (scopes.isEmpty()) return false

        val normalizedPath = path.replace('\\', '/').trimStart('/')
        if (normalizedPath.isEmpty()) return false

        return scopes.any { scope ->
            val relative = stripPrefix(normalizedPath, scope.baseDir) ?: return@any false
            scope.patterns.any { regex ->
                regex.matches(relative)
            }
        }
    }

    /**
     * Check if a directory name should be ignored (for early pruning during
     * traversal). Tests the directory's full relative path against all scopes.
     *
     * @param dirName The directory name (not full path)
     * @return true if the directory name matches a pattern that would ignore
     *         its contents
     */
    fun shouldIgnoreDirectory(dirName: String): Boolean {
        if (scopes.isEmpty()) return false
        return scopes.any { scope ->
            scope.patterns.any { pattern ->
                pattern.matches(dirName) ||
                    pattern.containsMatchIn("$dirName/")
            }
        }
    }

    /**
     * Return a new IgnorePatterns that adds a nested scope. The new scope's
     * patterns apply only to paths under [baseDir] (which must be relative to
     * the index root, without a leading slash). The returned instance retains
     * all existing scopes so parent patterns continue to apply within the
     * subtree.
     */
    fun plus(baseDir: String, patterns: List<String>): IgnorePatterns {
        val normalizedBase = baseDir.replace('\\', '/').trim('/').trimStart('/')
        val newScope = Scope(normalizedBase, patterns.compile())
        return IgnorePatterns(scopes + newScope, IgnorePatternsMarker)
    }

    /**
     * Return a new IgnorePatterns that adds a nested scope from raw file
     * content (the body of a =.arcologyignore= file).
     */
    fun plus(baseDir: String, content: String): IgnorePatterns =
        plus(baseDir, content.lines())

    private fun stripPrefix(path: String, baseDir: String): String? {
        if (baseDir.isEmpty()) return path
        // baseDir is a directory relative to root, no leading/trailing slash.
        // Match either "$baseDir/$rest" or exactly "$baseDir" (the dir itself).
        if (path == baseDir) return ""
        val prefix = "$baseDir/"
        if (!path.startsWith(prefix)) return null
        return path.substring(prefix.length)
    }

    companion object {
        /** Empty ignore patterns (ignores nothing) */
        val EMPTY = IgnorePatterns(emptyList<Scope>(), IgnorePatternsMarker)

        /**
         * Parse ignore patterns from file content, rooted at the index root.
         *
         * @param content The content of a =.arcologyignore= file
         * @return IgnorePatterns instance with a single root-scoped entry
         */
        fun parse(content: String): IgnorePatterns = IgnorePatterns(content.lines())

        /**
         * Build an IgnorePatterns rooted at a specific directory (used by the
         * walkers when they encounter a nested ignore file in isolation).
         */
        fun scoped(baseDir: String, content: String): IgnorePatterns =
            EMPTY.plus(baseDir, content)

        /**
         * Compile a list of raw pattern lines into regexes, dropping comments
         * and blank lines.
         */
        private fun List<String>.compile(): List<Regex> =
            filter { it.isNotBlank() && !it.startsWith("#") }
                .map { it.trim() }
                .map { compileGlobToRegex(it) }

        /**
         * Convert a glob pattern to a regex.
         *
         * Glob syntax:
         * - `**` matches anything including `/` (directory separator)
         * - `*` matches anything except `/`
         * - `?` matches any single character except `/`
         * - All other regex metacharacters are escaped
         *
         * The resulting regex matches the entire remainder string (anchored),
         * or any sub-path within it when the glob itself uses `**`.
         */
        private fun compileGlobToRegex(glob: String): Regex {
            val regex = StringBuilder()
            var i = 0

            while (i < glob.length) {
                when {
                    // Handle ** (matches across directories)
                    glob.startsWith("**", i) -> {
                        regex.append(".*")
                        i += 2
                        // Skip following / if present
                        if (i < glob.length && glob[i] == '/') {
                            regex.append("/?")
                            i++
                        }
                    }
                    // Handle * (matches within a single path component)
                    glob[i] == '*' -> {
                        regex.append("[^/]*")
                        i++
                    }
                    // Handle ? (matches single character except /)
                    glob[i] == '?' -> {
                        regex.append("[^/]")
                        i++
                    }
                    // Escape regex metacharacters
                    glob[i] in "\\.[]{}()+^\$|" -> {
                        regex.append("\\")
                        regex.append(glob[i])
                        i++
                    }
                    // Normal characters
                    else -> {
                        regex.append(glob[i])
                        i++
                    }
                }
            }

            // Pattern is anchored to the base dir: match the whole remainder,
            // or any sub-path of it (so a directory pattern prunes its contents).
            return Regex("(^|/)${regex}(/|\$)")
        }
    }
}

package computer.whatthefuck.arcology.indexer

import kotlin.test.Test
import kotlin.test.assertTrue
import kotlin.test.assertFalse

class IgnorePatternsTest {

    @Test
    fun `empty patterns should ignore nothing`() {
        val patterns = IgnorePatterns.EMPTY
        assertFalse(patterns.shouldIgnore("anything.org"))
        assertFalse(patterns.shouldIgnore("dir/file.org"))
    }

    @Test
    fun `should ignore files matching simple glob at root only`() {
        val patterns = IgnorePatterns(listOf("*.org_archive"))
        assertTrue(patterns.shouldIgnore("notes.org_archive"))
        // Anchored: a bare glob only matches at the root scope, not in subdirs.
        assertFalse(patterns.shouldIgnore("dir/notes.org_archive"))
        assertFalse(patterns.shouldIgnore("notes.org"))
    }

    @Test
    fun `should ignore direct children of a directory pattern`() {
        val patterns = IgnorePatterns(listOf("archive/*"))
        assertTrue(patterns.shouldIgnore("archive/old.org"))
        assertTrue(patterns.shouldIgnore("archive/subdir"))
        // Anchored: archive/* does not reach archive/sub/old.org.
        assertFalse(patterns.shouldIgnore("archive/sub/old.org"))
        assertFalse(patterns.shouldIgnore("notes/current.org"))
    }

    @Test
    fun `should ignore nested contents with double-star`() {
        val patterns = IgnorePatterns(listOf("archive/**"))
        assertTrue(patterns.shouldIgnore("archive/old.org"))
        assertTrue(patterns.shouldIgnore("archive/sub/old.org"))
        assertFalse(patterns.shouldIgnore("notes/current.org"))
    }

    @Test
    fun `should ignore specific subdirectory`() {
        val patterns = IgnorePatterns(listOf("daily/2023/*"))
        assertTrue(patterns.shouldIgnore("daily/2023/jan.org"))
        assertFalse(patterns.shouldIgnore("daily/2024/jan.org"))
        assertFalse(patterns.shouldIgnore("daily/notes.org"))
    }

    @Test
    fun `should handle double star glob`() {
        val patterns = IgnorePatterns(listOf("**/node_modules/**"))
        assertTrue(patterns.shouldIgnore("node_modules/foo"))
        assertTrue(patterns.shouldIgnore("src/node_modules/bar"))
        assertFalse(patterns.shouldIgnore("src/code.org"))
    }

    @Test
    fun `should skip comments and blank lines`() {
        val patterns = IgnorePatterns(listOf(
            "# This is a comment",
            "",
            "  ",
            "archive/*"
        ))
        assertTrue(patterns.shouldIgnore("archive/old.org"))
        assertFalse(patterns.shouldIgnore("notes.org"))
    }

    @Test
    fun `should parse from file content`() {
        val content = """
            # Arcology ignore file
            archive/*
            *.org_archive
            daily/2023/*
        """.trimIndent()

        val patterns = IgnorePatterns.parse(content)
        assertTrue(patterns.shouldIgnore("archive/old.org"))
        assertTrue(patterns.shouldIgnore("notes.org_archive"))
        assertTrue(patterns.shouldIgnore("daily/2023/jan.org"))
        assertFalse(patterns.shouldIgnore("notes.org"))
    }

    @Test
    fun `should handle question mark wildcard`() {
        val patterns = IgnorePatterns(listOf("temp?.org"))
        assertTrue(patterns.shouldIgnore("temp1.org"))
        assertTrue(patterns.shouldIgnore("tempX.org"))
        assertFalse(patterns.shouldIgnore("temp12.org"))
        assertFalse(patterns.shouldIgnore("temp.org"))
    }

    @Test
    fun `should handle multiple patterns`() {
        val patterns = IgnorePatterns(listOf(
            "archive/*",
            "*.bak",
            "tmp/**"
        ))
        assertTrue(patterns.shouldIgnore("archive/old.org"))
        assertTrue(patterns.shouldIgnore("file.bak"))
        assertTrue(patterns.shouldIgnore("tmp/deep/nested/file.org"))
        assertFalse(patterns.shouldIgnore("current/notes.org"))
    }

    @Test
    fun `should not ignore paths that don't match`() {
        val patterns = IgnorePatterns(listOf("archive/*"))
        assertFalse(patterns.shouldIgnore("notes.org"))
        assertFalse(patterns.shouldIgnore("src/main.org"))
        assertFalse(patterns.shouldIgnore("archiveX/file.org"))
    }

    @Test
    fun `nested scope only matches within its subtree`() {
        // A nested .arcologyignore in arroyo-system/ containing "template"
        val patterns = IgnorePatterns.EMPTY.plus("arroyo-system", listOf("template"))
        assertTrue(patterns.shouldIgnore("arroyo-system/template"))
        // Anchored: only the direct child of arroyo-system matches.
        assertFalse(patterns.shouldIgnore("arroyo-system/sub/template"))
        assertFalse(patterns.shouldIgnore("other/template"))
        assertFalse(patterns.shouldIgnore("template"))
    }

    @Test
    fun `nested scope with double-star matches deeper descendants`() {
        val patterns = IgnorePatterns.EMPTY.plus("arroyo-system", listOf("**/template"))
        assertTrue(patterns.shouldIgnore("arroyo-system/template"))
        assertTrue(patterns.shouldIgnore("arroyo-system/sub/template"))
        assertTrue(patterns.shouldIgnore("arroyo-system/a/b/template"))
        assertFalse(patterns.shouldIgnore("other/template"))
    }

    @Test
    fun `nested scope accumulates with root scope`() {
        val root = IgnorePatterns(listOf("*.bak"))
        val combined = root.plus("arroyo-system", listOf("template"))
        // Root '*.bak' is anchored to the root; it matches top-level .bak
        // files but not nested ones (use '**/*.bak' for that).
        assertTrue(combined.shouldIgnore("anywhere.bak"))
        assertFalse(combined.shouldIgnore("arroyo-system/notes.bak"))
        // Nested pattern applies only within arroyo-system.
        assertTrue(combined.shouldIgnore("arroyo-system/template"))
        assertFalse(combined.shouldIgnore("other/template"))
    }

    @Test
    fun `nested scope parsed from file content`() {
        val nested = IgnorePatterns.EMPTY.plus("arroyo-system", """
            # arroyo-system ignore
            template
            build/**
        """.trimIndent())
        assertTrue(nested.shouldIgnore("arroyo-system/template"))
        assertTrue(nested.shouldIgnore("arroyo-system/build/output.org"))
        assertFalse(nested.shouldIgnore("arroyo-system/build"))
        assertFalse(nested.shouldIgnore("other/template"))
    }

}

package computer.whatthefuck.arcology.indexer

import computer.whatthefuck.arcology.domain.AttachmentType
import computer.whatthefuck.arcology.domain.OrgAttachment

class AttachmentResolver(private val fileSystem: FileSystemInterface) {

    private val imageExtensions = setOf(
        "jpg", "jpeg", "png", "gif", "webp", "svg", "heic", "heif", "bmp", "tiff", "tif"
    )
    private val videoExtensions = setOf(
        "mp4", "webm", "mkv", "avi", "mov", "m4v"
    )

    suspend fun resolveAttachments(nodeId: String, filePath: String): List<OrgAttachment> {
        val paths = computeAttachmentPaths(nodeId)
        for (relativeDir in paths) {
            val files = fileSystem.listFilesInDirectory(filePath, relativeDir)
            if (files.isNotEmpty()) {
                return files.map { path ->
                    val filename = path.substringAfterLast("/")
                    OrgAttachment(
                        nodeId = nodeId,
                        resolvedPath = path,
                        type = classifyByExtension(filename)
                    )
                }
            }
        }
        return emptyList()
    }

    fun classifyByExtension(filename: String): AttachmentType {
        val ext = filename.substringAfterLast(".", "").lowercase()
        return when {
            ext in imageExtensions -> AttachmentType.IMAGE
            ext in videoExtensions -> AttachmentType.VIDEO
            else -> AttachmentType.FILE
        }
    }

    internal fun computeAttachmentPaths(nodeId: String): List<String> {
        if (nodeId.length < 2) return emptyList()
        val paths = mutableListOf<String>()
        if (nodeId.length >= 6) {
            paths.add("data/${nodeId.take(6)}/${nodeId.drop(6)}")
        }
        paths.add("data/${nodeId.take(2)}/${nodeId.drop(2)}")
        return paths
    }

    /**
     * Resolve a single attachment filename to its full path under the org-roam root.
     * Two-level nesting convention:
     *   1. data/<nodeId[0..1]>/<nodeId[2..]>/<filename>
     *   2. data/<nodeId[0..5]>/<nodeId[6..]>/<filename>
     *
     * @param nodeId The org node ID
     * @param filename The attachment filename
     * @param orgRoamRoot Root directory of the org-roam repository (may be empty for tree-URI-based filesystems)
     * @return The resolved path if found, null otherwise
     */
    suspend fun resolveAttachFile(
        nodeId: String,
        filename: String,
        orgRoamRoot: String = ""
    ): String? {
        if (nodeId.length < 2) return null
        val paths = computeAttachmentPaths(nodeId)
        for (relativeDir in paths) {
            val fullPath = if (orgRoamRoot.isEmpty()) {
                "$relativeDir/$filename"
            } else {
                "$orgRoamRoot/$relativeDir/$filename"
            }
            if (fileSystem.fileExists(fullPath)) return fullPath
        }
        return null
    }
}

<<ix-contract-preamble>>
<<ix-contract-interface>>
<<ix-contract-factory>>
<<ix-contract-fs>>
<<ix-contract-results>>

<<ix-progress-preamble>>
<<ix-progress-events>>
<<ix-progress-config>>

<<ix-ignore-preamble>>
<<ix-ignore-class>>

<<ix-ignore-test-prelude>>
<<ix-ignore-test>>
<<ix-ignore-test-end>>

<<ix-attach-preamble>>
<<ix-attach-class>>

<<ix-flow-preamble>>
<<ix-flow-class>>
<<ix-flow-pipeline>>
<<ix-flow-discover>>
<<ix-flow-load-ignore>>
<<ix-flow-index-file>>
<<ix-flow-record-failure>>
<<ix-flow-store-batched>>
<<ix-flow-deferred-fts>>
<<ix-flow-index-dir>>
<<ix-flow-index-single>>
<<ix-flow-remove-file>>
<<ix-flow-should-reindex>>
<<ix-flow-memory>>
<<ix-flow-closing>>

<<ix-cli-preamble>>
<<ix-cli-index>>
<<ix-cli-search>>
<<ix-cli-stats>>
<<ix-cli-parse-file>>