package computer.whatthefuck.arcology.indexer

import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.database.RoamRepositoryImpl
import computer.whatthefuck.arcology.database.QuizRepository
import computer.whatthefuck.arcology.database.QuizRepositoryImpl
import computer.whatthefuck.arcology.database.DatabaseFactory
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.flashcard.QuizIndexerPlugin
import computer.whatthefuck.arcology.agenda.TaskIndexerPlugin
import computer.whatthefuck.arcology.database.AgendaRepositoryImpl
import computer.whatthefuck.arroyo.ArroyoRepositoryImpl
import computer.whatthefuck.arcology.publishing.PublishingRepositoryImpl
import computer.whatthefuck.arcology.publishing.FeedRepositoryImpl
import computer.whatthefuck.arcology.publishing.ArcologyPublishingPlugin
import computer.whatthefuck.arcology.publishing.ArcologyAttachmentPlugin
import computer.whatthefuck.arcology.publishing.ArcologyFeedPlugin
import computer.whatthefuck.arcology.publishing.AttachmentCrusher

/**
 * JVM implementation of FileIndexingServiceFactory that creates FlowFileIndexer instances
 * with proper dependency injection for different environments.
 */
class FlowFileIndexerFactory(
    private val databasePath: String = "arcology.db",
    private val basePath: String? = null
) : FileIndexingServiceFactory {

    override fun createIndexingService(config: IndexingConfig): FileIndexingService {
        val database = DatabaseFactory.createDatabase(databasePath)
        val repository = RoamRepositoryImpl(database)
        val fileSystem = if (basePath != null) {
            JvmFileSystem(java.nio.file.Paths.get(basePath))
        } else {
            JvmFileSystem()
        }
        val parser = OrgFileParser()

        val publishingRepository = PublishingRepositoryImpl(database)
        val feedRepository = FeedRepositoryImpl(database)
        val plugins = listOf(
            QuizIndexerPlugin(QuizRepositoryImpl(database)),
            TaskIndexerPlugin(AgendaRepositoryImpl(database)),
            computer.whatthefuck.arroyo.ArroyoIndexerPlugin(ArroyoRepositoryImpl(database)),
            ArcologyPublishingPlugin(publishingRepository),
            ArcologyFeedPlugin(feedRepository),
            ArcologyAttachmentPlugin(
                repository = publishingRepository,
                crusher = AttachmentCrusher(AttachmentCrusher.defaultCacheDir()),
                attachmentResolver = AttachmentResolver(fileSystem),
                orgRoot = basePath ?: "."
            )
        )

        return FlowFileIndexer(
            repository = repository,
            plugins = plugins,
            parser = parser,
            fileSystem = fileSystem,
            config = config
        )
    }

    override fun createTestIndexingService(
        mockRepository: RoamRepository,
        mockFileSystem: FileSystemInterface,
        config: IndexingConfig
    ): FileIndexingService {
        val parser = OrgFileParser()

        return FlowFileIndexer(
            repository = mockRepository,
            parser = parser,
            fileSystem = mockFileSystem,
            config = config
        )
    }
}

/**
 * Extension function to create a FileIndexingService with default configuration
 */
fun createDefaultIndexingService(databasePath: String = "arcology.db"): FileIndexingService {
    return FlowFileIndexerFactory(databasePath).createIndexingService()
}

/**
 * Extension function to create a FileIndexingService with custom configuration
 */
fun createIndexingService(
    databasePath: String = "arcology.db",
    config: IndexingConfig,
    basePath: String? = null
): FileIndexingService {
    return FlowFileIndexerFactory(databasePath, basePath).createIndexingService(config)
}

package computer.whatthefuck.arcology.indexer

import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.emptyFlow
import kotlinx.coroutines.flow.flowOn
import kotlinx.datetime.Instant
import java.io.File
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import kotlin.io.path.*
import kotlin.io.path.readBytes

/**
 * JVM implementation of FileSystemInterface using Java NIO
 * 
 * @param basePath The root directory for org files. All paths are stored relative to this base.
 */
class JvmFileSystem(
    private val basePath: Path? = null
) : FileSystemInterface {

    /**
     * Convert a relative path to an absolute path using the base path.
     */
    private fun toAbsolutePath(relativePath: String): Path {
        return if (basePath != null) {
            basePath.resolve(relativePath)
        } else {
            Path(relativePath)
        }
    }

    /**
     * Convert an absolute path to a relative path by stripping the base path prefix.
     */
    fun toRelativePath(absolutePath: String): String {
        return if (basePath != null) {
            val absolute = Path(absolutePath)
            if (absolute.startsWith(basePath)) {
                basePath.relativize(absolute).toString()
            } else {
                absolutePath
            }
        } else {
            absolutePath
        }
    }

    override suspend fun fileExists(path: String): Boolean {
        val absolutePath = toAbsolutePath(path)
        return absolutePath.exists()
    }

    override suspend fun readIgnoreFile(rootPath: String): String? {
        val root = if (basePath != null) basePath else Path(rootPath)
        val ignoreFile = root.resolve(".arcologyignore")
        return if (ignoreFile.exists() && ignoreFile.isRegularFile()) {
            ignoreFile.readText()
        } else {
            null
        }
    }

    override suspend fun readFile(path: String): String {
        val absolutePath = toAbsolutePath(path)
        return absolutePath.readText()
    }

    override suspend fun readFileBytes(path: String): ByteArray {
        val absolutePath = toAbsolutePath(path)
        return absolutePath.readBytes()
    }

    override suspend fun writeFile(path: String, content: String) {
        val absolutePath = toAbsolutePath(path)
        absolutePath.writeText(content)
    }

    /**
     * True append using [java.nio.file.Files.write] with
     * [StandardOpenOption.APPEND] and [StandardOpenOption.CREATE]. This is
     * O(content) rather than the O(file+content) read-merge-write default,
     * which matters for the org-fc review-history TSV (tens of thousands of
     * rows, one row appended per review).
     */
    override suspend fun appendToFile(path: String, content: String) {
        val absolutePath = toAbsolutePath(path)
        // Use java.nio.file.Files directly to avoid the writeText(charset)
        // overload ambiguity; this gives a true O(content) append.
        java.nio.file.Files.write(
            absolutePath,
            content.toByteArray(Charsets.UTF_8),
            StandardOpenOption.APPEND, StandardOpenOption.CREATE
        )
    }

    override suspend fun getLastModified(path: String): Instant {
        val absolutePath = toAbsolutePath(path)
        val lastModified = absolutePath.getLastModifiedTime().toInstant()
        return Instant.fromEpochSeconds(
            lastModified.epochSecond,
            lastModified.nano
        )
    }

    override fun listOrgFiles(path: String, recursive: Boolean): Flow<String> {
        return listOrgFiles(path, recursive, IgnorePatterns.EMPTY)
    }

    override fun listOrgFiles(path: String, recursive: Boolean, ignorePatterns: IgnorePatterns): Flow<String> {
        val base = if (basePath != null) basePath else Path(path)

        if (!base.exists()) {
            return emptyFlow()
        }

        val pathSequence = if (recursive) {
            walkWithIgnorePatterns(base, ignorePatterns)
        } else if (base.isDirectory()) {
            base.listDirectoryEntries().asSequence()
                .filter { file ->
                    val relativePath = base.relativize(file).toString()
                    !ignorePatterns.shouldIgnore(relativePath)
                }
        } else {
            sequenceOf(base)
        }

        return pathSequence
            .filter { it.isRegularFile() }
            .filter { it.extension.lowercase() == "org" }
            .filter { !it.fileName.toString().startsWith(".#") }
            .map { 
                val absolutePath = it.absolutePathString()
                toRelativePath(absolutePath)
            }
            .asFlow()
            .flowOn(Dispatchers.IO)
    }

    override suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> {
        val absoluteBase = toAbsolutePath(baseFilePath)
        val parentDir = absoluteBase.parent ?: return emptyList()
        val targetDir = parentDir.resolve(relativeDir)
        if (!targetDir.exists() || !targetDir.isDirectory()) return emptyList()
        return targetDir.listDirectoryEntries()
            .filter { it.isRegularFile() }
            .map { 
                val absolutePath = it.absolutePathString()
                toRelativePath(absolutePath)
            }
    }

    /**
     * Walk a directory tree while respecting ignore patterns.
     * Prunes ignored directories early to avoid traversing them.
     *
     * As the walk descends into subdirectories, any =.arcologyignore= file
     * found in a directory is read and merged into the inherited
     * IgnorePatterns as a new scope anchored at that directory's path
     * relative to the index root. This supports nested ignore files that
     * only apply within their subtree.
     */
    private fun walkWithIgnorePatterns(basePath: Path, ignorePatterns: IgnorePatterns): Sequence<Path> = sequence {
        // Each stack entry carries the directory to visit and the accumulated
        // IgnorePatterns that apply within that directory.
        val stack = ArrayDeque<Pair<Path, IgnorePatterns>>()
        stack.addLast(basePath to ignorePatterns)

        while (stack.isNotEmpty()) {
            val (current, currentPatterns) = stack.removeLast()
            val relativePath = basePath.relativize(current).toString()

            // Skip if this path matches an ignore pattern
            if (relativePath.isNotEmpty() && currentPatterns.shouldIgnore(relativePath)) {
                continue
            }

            if (current.isDirectory()) {
                try {
                    val children = current.listDirectoryEntries()

                    // Check for a nested .arcologyignore in this directory and
                    // accumulate its patterns as a new scope anchored here.
                    val childPatterns = try {
                        val ignoreFile = current.resolve(".arcologyignore")
                        if (ignoreFile.exists() && ignoreFile.isRegularFile()) {
                            currentPatterns.plus(relativePath, ignoreFile.readText())
                        } else {
                            currentPatterns
                        }
                    } catch (e: Exception) {
                        // If we can't read the ignore file, fall back to the
                        // inherited patterns rather than aborting the walk.
                        currentPatterns
                    }

                    // Push children with the accumulated patterns so nested
                    // scopes apply within the subtree.
                    children.forEach { child ->
                        stack.addLast(child to childPatterns)
                    }
                } catch (e: Exception) {
                    // Skip directories we can't read
                }
            } else {
                yield(current)
            }
        }
    }

    override suspend fun copyDatabase(sourcePath: String, targetPath: String): Boolean {
        return try {
            val source = Path(sourcePath)
            val target = Path(targetPath)
            source.copyTo(target, overwrite = true)
            true
        } catch (e: Exception) {
            println("Failed to copy database from $sourcePath to $targetPath: ${e.message}")
            false
        }
    }
}

package computer.whatthefuck.arcology.indexer

import android.content.ContentResolver
import android.content.Context
import android.net.Uri
import android.provider.DocumentsContract
import android.provider.DocumentsContract.Document
import android.provider.OpenableColumns
import android.util.Log
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.FlowCollector
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.withContext
import kotlinx.datetime.Instant
import java.io.InputStream
import java.io.OutputStream
import java.util.concurrent.ConcurrentHashMap

private const val TAG = "AndroidFileSystem"

/**
 * Cached file metadata from ContentProvider query.
 * Storing this during discovery avoids repeated queries later.
 */
data class FileMetadataCache(
    val documentId: String,
    val displayName: String,
    val lastModified: Long,
    val uri: Uri
)

/**
 * Android implementation of FileSystemInterface using Storage Access Framework.
 * Uses direct ContentResolver queries for performance (10x faster than DocumentFile).
 *
 * @param context Application context for ContentResolver access
 * @param treeUri The tree URI from ACTION_OPEN_DOCUMENT_TREE result
 * @param treeDocumentId The document ID of the tree root, used for path conversion
 * @throws IllegalArgumentException if treeUri is null, empty, or invalid
 */
class AndroidFileSystem(
    private val context: Context,
    private val treeUri: Uri,
    private val treeDocumentId: String? = null
) : FileSystemInterface {

    init {
        val uriString = treeUri.toString()
        require(uriString.isNotEmpty() && treeUri != Uri.EMPTY) {
            "AndroidFileSystem requires a valid tree URI. Got: $treeUri. " +
            "This usually means AppPreferences.getSelectedDirectoryUri() returned null. " +
            "Ensure the user has selected a directory via ACTION_OPEN_DOCUMENT_TREE."
        }
    }

    private val contentResolver get() = context.contentResolver

    /**
     * Extract the tree document ID from a tree URI.
     * This is the base path for all relative path conversions.
     */
    private fun getTreeDocumentId(): String {
        return treeDocumentId ?: DocumentsContract.getTreeDocumentId(treeUri)
    }

    /**
     * Convert a relative path to an absolute content URI.
     * Example: "notes/project.org" → content://.../document/primary%3Aorg-roam%2Fnotes%2Fproject.org
     */
    fun toAbsoluteUri(relativePath: String): String {
        val baseDocId = getTreeDocumentId()
        val fullDocId = if (relativePath.isEmpty()) {
            baseDocId
        } else {
            "$baseDocId/$relativePath"
        }
        return DocumentsContract.buildDocumentUriUsingTree(treeUri, fullDocId).toString()
    }

    /**
     * Convert an absolute content URI to a relative path.
     * Example: content://.../document/primary%3Aorg-roam%2Fnotes%2Fproject.org → "notes/project.org"
     */
    fun toRelativePath(absoluteUri: String): String {
        return try {
            val uri = Uri.parse(absoluteUri)
            val docId = DocumentsContract.getDocumentId(uri)
            val baseDocId = getTreeDocumentId()
            
            if (docId == baseDocId) {
                ""
            } else if (docId.startsWith("$baseDocId/")) {
                docId.substringAfter("$baseDocId/")
            } else {
                absoluteUri
            }
        } catch (e: Exception) {
            Log.e(TAG, "toRelativePath: failed for $absoluteUri", e)
            absoluteUri
        }
    }

    /**
     * Check if a database export exists in the root directory.
     * @return URI to arcology.db if found, null otherwise
     */
    suspend fun checkForDatabaseExport(): Uri? = withContext(Dispatchers.IO) {
        try {
            Log.d(TAG, "checkForDatabaseExport: searching for arcology.db in root")
            val rootDocId = getTreeDocumentId()
            val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, rootDocId)

            contentResolver.query(
                childrenUri,
                arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_DISPLAY_NAME),
                null, null, null
            )?.use { cursor ->
                val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
                val nameIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DISPLAY_NAME)

                while (cursor.moveToNext()) {
                    val name = cursor.getString(nameIdx)
                    if (name == "arcology.db") {
                        val docId = cursor.getString(idIdx)
                        val dbUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, docId)
                        Log.d(TAG, "checkForDatabaseExport: found arcology.db at $dbUri")
                        return@withContext dbUri
                    }
                }
            }
            Log.d(TAG, "checkForDatabaseExport: arcology.db not found")
            null
        } catch (e: Exception) {
            Log.e(TAG, "checkForDatabaseExport: error", e)
            null
        }
    }

    // Cache of file metadata discovered during listOrgFiles()
    // Key is the file URI string, value is the cached metadata
    private val metadataCache = ConcurrentHashMap<String, FileMetadataCache>()

    // Projection for efficient ContentResolver queries - fetch all needed columns in one query
    private val DOCUMENT_PROJECTION = arrayOf(
        Document.COLUMN_DOCUMENT_ID,
        Document.COLUMN_DISPLAY_NAME,
        Document.COLUMN_MIME_TYPE,
        Document.COLUMN_LAST_MODIFIED
    )

    override suspend fun fileExists(path: String): Boolean = withContext(Dispatchers.IO) {
        Log.d(TAG, "fileExists: checking $path")
        try {
            // Check cache first
            val cached = metadataCache[path]
            if (cached != null) {
                Log.d(TAG, "fileExists: found in cache")
                return@withContext true
            }

            // Convert relative path to absolute URI if needed
            val uri = if (path.startsWith("content://")) {
                Uri.parse(path)
            } else {
                Uri.parse(toAbsoluteUri(path))
            }
            
            // Use a lightweight query to check existence instead of DocumentFile
            contentResolver.query(uri, arrayOf(Document.COLUMN_DOCUMENT_ID), null, null, null)?.use { cursor ->
                val exists = cursor.moveToFirst()
                Log.d(TAG, "fileExists: query result=$exists for $path")
                return@withContext exists
            }
            Log.d(TAG, "fileExists: query returned null for $path")
            false
        } catch (e: Exception) {
            Log.e(TAG, "fileExists: exception for $path", e)
            false
        }
    }

    override suspend fun readFile(path: String): String = withContext(Dispatchers.IO) {
        Log.d(TAG, "readFile: starting for $path")
        val uri = if (path.startsWith("content://")) {
            Uri.parse(path)
        } else {
            Uri.parse(toAbsoluteUri(path))
        }
        Log.d(TAG, "readFile: opening input stream for $uri")
        val content = contentResolver.openInputStream(uri)?.use { stream ->
            Log.d(TAG, "readFile: stream opened, reading text")
            val text = stream.bufferedReader().readText()
            Log.d(TAG, "readFile: read ${text.length} chars from $path")
            text
        } ?: throw IllegalStateException("Cannot open file: $path")
        Log.d(TAG, "readFile: complete for $path")
        content
    }

    override suspend fun readFileBytes(path: String): ByteArray = withContext(Dispatchers.IO) {
        Log.d(TAG, "readFileBytes: starting for $path")
        val uri = if (path.startsWith("content://")) {
            Uri.parse(path)
        } else {
            Uri.parse(toAbsoluteUri(path))
        }
        Log.d(TAG, "readFileBytes: opening input stream for $uri")
        val content = contentResolver.openInputStream(uri)?.use { stream ->
            Log.d(TAG, "readFileBytes: stream opened, reading bytes")
            stream.readBytes()
        } ?: throw IllegalStateException("Cannot open file: $path")
        Log.d(TAG, "readFileBytes: complete for $path, ${content.size} bytes")
        content
    }

    override suspend fun getLastModified(path: String): Instant = withContext(Dispatchers.IO) {
        Log.d(TAG, "getLastModified: starting for $path")

        // Check cache first - this avoids a ContentProvider roundtrip
        val cached = metadataCache[path]
        if (cached != null) {
            Log.d(TAG, "getLastModified: using cached value ${cached.lastModified} for $path")
            return@withContext Instant.fromEpochMilliseconds(cached.lastModified)
        }

        // Convert relative path to absolute URI if needed
        val uri = if (path.startsWith("content://")) {
            Uri.parse(path)
        } else {
            Uri.parse(toAbsoluteUri(path))
        }
        
        // Fallback to querying the ContentProvider
        val lastModified = contentResolver.query(
            uri,
            arrayOf(Document.COLUMN_LAST_MODIFIED),
            null, null, null
        )?.use { cursor ->
            if (cursor.moveToFirst()) {
                val idx = cursor.getColumnIndexOrThrow(Document.COLUMN_LAST_MODIFIED)
                cursor.getLong(idx)
            } else {
                0L
            }
        } ?: 0L

        Log.d(TAG, "getLastModified: got timestamp $lastModified for $path")
        Instant.fromEpochMilliseconds(lastModified)
    }

    override suspend fun readIgnoreFile(rootPath: String): String? = withContext(Dispatchers.IO) {
        Log.d(TAG, "readIgnoreFile: looking for .arcologyignore in tree root")
        try {
            val rootDocId = DocumentsContract.getTreeDocumentId(treeUri)
            val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, rootDocId)

            // Query for .arcologyignore file in root
            contentResolver.query(
                childrenUri,
                arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_DISPLAY_NAME),
                null, null, null
            )?.use { cursor ->
                val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
                val nameIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DISPLAY_NAME)

                while (cursor.moveToNext()) {
                    val name = cursor.getString(nameIdx)
                    if (name == ".arcologyignore") {
                        val docId = cursor.getString(idIdx)
                        val ignoreUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, docId)

                        // Read the file content
                        return@withContext contentResolver.openInputStream(ignoreUri)?.use { stream ->
                            stream.bufferedReader().readText()
                        }
                    }
                }
            }
            Log.d(TAG, "readIgnoreFile: .arcologyignore not found")
            null
        } catch (e: Exception) {
            Log.e(TAG, "readIgnoreFile: error reading ignore file", e)
            null
        }
    }

    override fun listOrgFiles(path: String, recursive: Boolean): Flow<String> {
        return listOrgFiles(path, recursive, IgnorePatterns.EMPTY)
    }

    override fun listOrgFiles(path: String, recursive: Boolean, ignorePatterns: IgnorePatterns): Flow<String> = flow {
        Log.d(TAG, "listOrgFiles: starting efficient ContentResolver traversal")
        // Clear metadata cache for fresh discovery
        metadataCache.clear()

        val rootDocId = DocumentsContract.getTreeDocumentId(treeUri)
        emitOrgFilesEfficient(rootDocId, "", recursive, ignorePatterns)
    }.flowOn(Dispatchers.IO)

    /**
     * Efficient file discovery using direct ContentResolver queries.
     * This is ~10x faster than DocumentFile.listFiles() because it:
     * 1. Fetches all metadata in a single query per directory
     * 2. Caches metadata for later use (avoiding repeated queries)
     * 3. Avoids the DocumentFile abstraction overhead
     * 4. Prunes ignored directories early (avoids traversing them)
     * 5. Reads nested .arcologyignore files as it descends and accumulates
     *    their patterns as a new scope anchored at the containing directory,
     *    so nested ignore files only apply within their subtree.
     *
     * @param parentDocId The document ID of the directory to scan
     * @param relativePath The relative path from root (for ignore pattern matching)
     * @param recursive Whether to recurse into subdirectories
     * @param ignorePatterns Patterns for filtering files and directories
     */
    private suspend fun FlowCollector<String>.emitOrgFilesEfficient(
        parentDocId: String,
        relativePath: String,
        recursive: Boolean,
        ignorePatterns: IgnorePatterns
    ) {
        val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, parentDocId)
        val directories = mutableListOf<Triple<String, String, String>>() // docId, name, fullPath
        var nestedIgnoreDocId: String? = null

        contentResolver.query(childrenUri, DOCUMENT_PROJECTION, null, null, null)?.use { cursor ->
            val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
            val nameIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DISPLAY_NAME)
            val mimeIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_MIME_TYPE)
            val modifiedIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_LAST_MODIFIED)

            while (cursor.moveToNext()) {
                val docId = cursor.getString(idIdx)
                val name = cursor.getString(nameIdx) ?: continue
                val mimeType = cursor.getString(mimeIdx)
                val lastModified = cursor.getLong(modifiedIdx)

                // Detect a nested .arcologyignore in this directory. It is
                // neither emitted as an org file nor subject to ignore
                // filtering itself; its content is read after the cursor
                // closes and merged as a new scope anchored at relativePath.
                if (name == ".arcologyignore" && mimeType != Document.MIME_TYPE_DIR) {
                    nestedIgnoreDocId = docId
                    continue
                }

                // Build full relative path for ignore pattern matching
                val fullPath = if (relativePath.isEmpty()) name else "$relativePath/$name"

                // Check ignore patterns early to avoid unnecessary work
                if (ignorePatterns.shouldIgnore(fullPath)) {
                    Log.d(TAG, "emitOrgFilesEfficient: ignoring $fullPath")
                    continue
                }

                if (mimeType == Document.MIME_TYPE_DIR) {
                    // Queue directories for later traversal (if recursive)
                    if (recursive) {
                        directories.add(Triple(docId, name, fullPath))
                    }
                } else if (name.endsWith(".org", ignoreCase = true) && !name.startsWith(".#")) {
                    // Build the file URI and convert to relative path
                    val fileUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, docId)
                    val uriString = fileUri.toString()

                    // Cache the metadata for later use (keyed by URI)
                    metadataCache[uriString] = FileMetadataCache(
                        documentId = docId,
                        displayName = name,
                        lastModified = lastModified,
                        uri = fileUri
                    )

                    // Emit relative path instead of absolute URI
                    val relativePath = toRelativePath(uriString)
                    emit(relativePath)
                }
            }
        }

        // Read any nested .arcologyignore and accumulate it as a new scope
        // anchored at this directory's relativePath before recursing.
        val childPatterns = if (nestedIgnoreDocId != null) {
            try {
                val ignoreUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, nestedIgnoreDocId)
                val content = contentResolver.openInputStream(ignoreUri)?.use { stream ->
                    stream.bufferedReader().readText()
                }
                if (content != null) {
                    ignorePatterns.plus(relativePath, content)
                } else {
                    ignorePatterns
                }
            } catch (e: Exception) {
                Log.e(TAG, "emitOrgFilesEfficient: error reading nested .arcologyignore at $relativePath", e)
                ignorePatterns
            }
        } else {
            ignorePatterns
        }

        // Recurse into directories with the accumulated patterns so nested
        // scopes apply within the subtree.
        for ((dirId, dirName, fullPath) in directories) {
            emitOrgFilesEfficient(dirId, fullPath, recursive, childPatterns)
        }
    }

    override suspend fun listFilesInDirectory(baseFilePath: String, relativeDir: String): List<String> = withContext(Dispatchers.IO) {
        try {
            // Get the document ID of the base file and derive its parent directory
            val baseUri = Uri.parse(baseFilePath)
            val baseDocId = DocumentsContract.getDocumentId(baseUri)
            // Parent document ID: drop the last path segment from the document ID
            val parentDocId = baseDocId.substringBeforeLast("/", baseDocId)

            // Walk down through each segment of relativeDir
            var currentDocId = parentDocId
            val segments = relativeDir.split("/").filter { it.isNotEmpty() }
            for (segment in segments) {
                val childDocId = findChildByName(currentDocId, segment)
                if (childDocId == null) {
                    return@withContext emptyList()
                }
                currentDocId = childDocId
            }

            // List non-directory children in the target directory
            val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, currentDocId)
            val result = mutableListOf<String>()

            contentResolver.query(
                childrenUri,
                arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_MIME_TYPE),
                null, null, null
            )?.use { cursor ->
                val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
                val mimeIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_MIME_TYPE)

                while (cursor.moveToNext()) {
                    val mimeType = cursor.getString(mimeIdx)
                    if (mimeType != Document.MIME_TYPE_DIR) {
                        val docId = cursor.getString(idIdx)
                        val fileUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, docId)
                        result.add(fileUri.toString())
                    }
                }
            }

            result
        } catch (e: Exception) {
            Log.e(TAG, "listFilesInDirectory: error for baseFilePath=$baseFilePath, relativeDir=$relativeDir", e)
            emptyList()
        }
    }

    /**
     * Clear the metadata cache. Call this if the underlying files may have changed.
     */
    fun clearMetadataCache() {
        metadataCache.clear()
    }

    /**
     * Get the current size of the metadata cache.
     */
    fun getMetadataCacheSize(): Int = metadataCache.size

    /**
     * Write content to a file, replacing existing content.
     */
    override suspend fun writeFile(path: String, content: String) = withContext(Dispatchers.IO) {
        val fileUri = if (path.startsWith("content://")) {
            Uri.parse(path)
        } else {
            Uri.parse(toAbsoluteUri(path))
        }
        contentResolver.openOutputStream(fileUri, "wt")?.use { stream ->
            stream.write(content.toByteArray(Charsets.UTF_8))
        } ?: throw IllegalStateException("Cannot open file for writing: $path")
    }

    /**
     * Append content to a file, creating it if it does not exist.
     *
     * Android SAF has no true append mode for DocumentFile-backed URIs that
     * is also create-if-absent, so we fall back to the read-merge-write
     * default for the relative-path case. The "wa" mode below works for
     * files that already exist; we guard with fileExists so a missing file
     * is created via writeFile instead.
     *
     * This is best-effort: concurrent appends from Emacs and the Android app
     * can interleave. Review frequency is low (one row per card review), so
     * the read-merge-write cost is acceptable in practice.
     */
    override suspend fun appendToFile(path: String, content: String) = withContext(Dispatchers.IO) {
        if (!fileExists(path)) {
            writeFile(path, content)
            return@withContext
        }
        val fileUri = if (path.startsWith("content://")) {
            Uri.parse(path)
        } else {
            Uri.parse(toAbsoluteUri(path))
        }
        contentResolver.openOutputStream(fileUri, "wa")?.use { stream ->
            stream.write(content.toByteArray(Charsets.UTF_8))
        } ?: throw IllegalStateException("Cannot open file for appending: $path")
    }

    /**
     * Create a new file under the given parent document ID.
     * Returns the URI string of the created file.
     */
    suspend fun createFile(parentDocId: String, fileName: String): String = withContext(Dispatchers.IO) {
        val parentUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, parentDocId)
        val mimeType = if (fileName.endsWith(".org")) "text/org" else "application/octet-stream"
        val createdUri = DocumentsContract.createDocument(contentResolver, parentUri, mimeType, fileName)
            ?: throw IllegalStateException("Failed to create file: $fileName")
        createdUri.toString()
    }

    /**
     * Find a child document by display name under the given parent document ID.
     * Returns the child's document ID, or null if not found.
     */
    suspend fun findChildByName(parentDocId: String, name: String): String? = withContext(Dispatchers.IO) {
        val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, parentDocId)
        contentResolver.query(
            childrenUri,
            arrayOf(Document.COLUMN_DOCUMENT_ID, Document.COLUMN_DISPLAY_NAME),
            null, null, null
        )?.use { cursor ->
            val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
            val nameIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DISPLAY_NAME)
            while (cursor.moveToNext()) {
                if (cursor.getString(nameIdx) == name) {
                    return@withContext cursor.getString(idIdx)
                }
            }
        }
        null
    }

    /**
     * Create a subdirectory under the given parent document ID.
     * Returns the document ID of the created directory.
     */
    suspend fun createDirectory(parentDocId: String, dirName: String): String = withContext(Dispatchers.IO) {
        val parentUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, parentDocId)
        val createdUri = DocumentsContract.createDocument(
            contentResolver, parentUri, Document.MIME_TYPE_DIR, dirName
        ) ?: throw IllegalStateException("Failed to create directory: $dirName")
        DocumentsContract.getDocumentId(createdUri)
    }

    /**
     * Build a document URI from a document ID using this file system's tree URI.
     */
    fun buildDocumentUri(docId: String): String {
        return DocumentsContract.buildDocumentUriUsingTree(treeUri, docId).toString()
    }

    /**
     * Get the root document ID of the tree.
     */
    fun getRootDocumentId(): String {
        return DocumentsContract.getTreeDocumentId(treeUri)
    }

    /**
     * Entry representing a file or directory in the file browser.
     */
    data class FileEntry(
        val documentId: String,
        val displayName: String,
        val isDirectory: Boolean,
        val lastModified: Long,
        val uri: Uri
    )

    /**
     * List all children (files and directories) of a directory.
     * Used for file browser navigation.
     */
    suspend fun listChildren(parentDocId: String): List<FileEntry> = withContext(Dispatchers.IO) {
        val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(treeUri, parentDocId)
        val entries = mutableListOf<FileEntry>()

        contentResolver.query(childrenUri, DOCUMENT_PROJECTION, null, null, null)?.use { cursor ->
            val idIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DOCUMENT_ID)
            val nameIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_DISPLAY_NAME)
            val mimeIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_MIME_TYPE)
            val modifiedIdx = cursor.getColumnIndexOrThrow(Document.COLUMN_LAST_MODIFIED)

            while (cursor.moveToNext()) {
                val docId = cursor.getString(idIdx)
                val name = cursor.getString(nameIdx) ?: continue
                val mimeType = cursor.getString(mimeIdx)
                val lastModified = cursor.getLong(modifiedIdx)

                // Skip hidden files (starting with .)
                if (name.startsWith(".")) continue

                val isDirectory = mimeType == Document.MIME_TYPE_DIR
                val fileUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, docId)

                entries.add(FileEntry(
                    documentId = docId,
                    displayName = name,
                    isDirectory = isDirectory,
                    lastModified = lastModified,
                    uri = fileUri
                ))
            }
        }

        // Sort: directories first, then by name
        entries.sortedWith(compareBy({ !it.isDirectory }, { it.displayName.lowercase() }))
    }

    /**
     * Copy a file from a source URI into a directory in the SAF tree.
     * Handles duplicate filenames by appending a counter (e.g., photo.jpg, photo (1).jpg).
     *
     * @param sourceUri The URI of the source file (content:// or file://)
     * @param targetParentDocId The document ID of the target directory
     * @param targetName The desired filename for the copied file
     * @return The URI string of the copied file, or null if the copy failed
     */
    suspend fun copyFileToTree(
        sourceUri: Uri,
        targetParentDocId: String,
        targetName: String
    ): String? = withContext(Dispatchers.IO) {
        try {
            Log.d(TAG, "copyFileToTree: copying $sourceUri to $targetParentDocId/$targetName")

            // Get MIME type of source file
            val sourceMimeType = getMimeType(sourceUri) ?: "application/octet-stream"

            // Find a unique filename if the target already exists
            val uniqueName = findUniqueFilename(targetParentDocId, targetName)

            // Create the target file
            val parentUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, targetParentDocId)
            val targetFileUri = DocumentsContract.createDocument(
                contentResolver, parentUri, sourceMimeType, uniqueName
            ) ?: throw IllegalStateException("Failed to create target file: $uniqueName")

            // Copy the file content
            contentResolver.openInputStream(sourceUri)?.use { input: InputStream ->
                contentResolver.openOutputStream(targetFileUri)?.use { output: OutputStream ->
                    input.copyTo(output)
                } ?: throw IllegalStateException("Failed to open output stream for: $targetFileUri")
            } ?: throw IllegalStateException("Failed to open input stream for: $sourceUri")

            Log.d(TAG, "copyFileToTree: successfully copied to $targetFileUri")
            targetFileUri.toString()
        } catch (e: Exception) {
            Log.e(TAG, "copyFileToTree: failed to copy file", e)
            null
        }
    }

    /**
     * Create a file with binary content in a directory.
     *
     * @param parentDocId The document ID of the parent directory
     * @param fileName The name of the file to create
     * @param mimeType The MIME type of the file
     * @param content The binary content to write
     * @return The URI string of the created file, or null if creation failed
     */
    suspend fun createFileWithContent(
        parentDocId: String,
        fileName: String,
        mimeType: String,
        content: ByteArray
    ): String? = withContext(Dispatchers.IO) {
        try {
            val uniqueName = findUniqueFilename(parentDocId, fileName)
            val parentUri = DocumentsContract.buildDocumentUriUsingTree(treeUri, parentDocId)
            val fileUri = DocumentsContract.createDocument(
                contentResolver, parentUri, mimeType, uniqueName
            ) ?: throw IllegalStateException("Failed to create file: $uniqueName")

            contentResolver.openOutputStream(fileUri)?.use { output ->
                output.write(content)
            } ?: throw IllegalStateException("Failed to open output stream for: $fileUri")

            fileUri.toString()
        } catch (e: Exception) {
            Log.e(TAG, "createFileWithContent: failed", e)
            null
        }
    }

    /**
     * Get the MIME type of a file from its URI.
     *
     * @param uri The URI of the file
     * @return The MIME type, or null if it couldn't be determined
     */
    fun getMimeType(uri: Uri): String? {
        return contentResolver.getType(uri)
    }

    /**
     * Find a unique filename by appending a counter if the file already exists.
     *
     * @param parentDocId The document ID of the parent directory
     * @param fileName The desired filename
     * @return A unique filename (may have a counter suffix)
     */
    private suspend fun findUniqueFilename(parentDocId: String, fileName: String): String = withContext(Dispatchers.IO) {
        var candidate = fileName
        var counter = 1

        val baseName = fileName.substringBeforeLast(".")
        val extension = if (fileName.contains(".")) ".${fileName.substringAfterLast(".")}" else ""

        while (findChildByName(parentDocId, candidate) != null) {
            candidate = "$baseName ($counter)$extension"
            counter++
        }

        candidate
    }

    /**
     * Get the file size from a URI.
     *
     * @param uri The URI of the file
     * @return The file size in bytes, or null if it couldn't be determined
     */
    fun getFileSize(uri: Uri): Long? {
        return try {
            contentResolver.openFileDescriptor(uri, "r")?.use { pfd ->
                pfd.statSize
            }
        } catch (e: Exception) {
            Log.e(TAG, "getFileSize: failed for $uri", e)
            null
        }
    }

    override suspend fun copyDatabase(sourcePath: String, targetPath: String): Boolean = withContext(Dispatchers.IO) {
        try {
            Log.d(TAG, "copyDatabase: copying from $sourcePath to $targetPath")
            
            // Source is a file path, target is a relative path in SAF tree
            val targetUri = toAbsoluteUri(targetPath)
            
            // Get the root document ID to find the target parent
            val rootDocId = getTreeDocumentId()
            
            // Copy using content resolver
            contentResolver.openInputStream(Uri.parse("file://$sourcePath"))?.use { inputStream ->
                contentResolver.openOutputStream(Uri.parse(targetUri), "wt")?.use { outputStream ->
                    inputStream.copyTo(outputStream)
                } ?: run {
                    Log.e(TAG, "copyDatabase: failed to open output stream for $targetUri")
                    return@withContext false
                }
            } ?: run {
                Log.e(TAG, "copyDatabase: failed to open input stream for $sourcePath")
                return@withContext false
            }
            
            Log.d(TAG, "copyDatabase: successfully copied to $targetUri")
            true
        } catch (e: Exception) {
            Log.e(TAG, "copyDatabase: failed", e)
            false
        }
    }
}

package computer.whatthefuck.arcology.indexer

import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import kotlin.io.path.createDirectories
import kotlin.io.path.createTempDirectory
import kotlin.io.path.pathString
import kotlin.io.path.writeText
import kotlin.test.Test
import kotlin.test.assertTrue
import kotlin.test.assertFalse

class JvmFileSystemNestedIgnoreTest {

    @Test
    fun `nested arcologyignore excludes only its subtree direct child`() = runBlocking {
        val root = createTempDirectory(prefix = "arcology-test-")
        try {
            // Root layout:
            //   root/a.org
            //   root/arroyo-system/.arcologyignore   (contains: template)
            //   root/arroyo-system/template/inside.org   (pruned by 'template')
            //   root/arroyo-system/keep.org
            //   root/arroyo-system/sub/template/inside.org  (NOT matched by bare 'template')
            //   root/other/template/inside.org             (outside the nested scope)
            root.resolve("a.org").writeText("* A\n")
            val sub = root.resolve("arroyo-system").createDirectories()
            sub.resolve(".arcologyignore").writeText("template\n")
            sub.resolve("template").createDirectories().resolve("inside.org").writeText("* inside\n")
            sub.resolve("keep.org").writeText("* keep\n")
            sub.resolve("sub").createDirectories().resolve("template").createDirectories().resolve("inside.org").writeText("* deep\n")
            root.resolve("other").createDirectories().resolve("template").createDirectories().resolve("inside.org").writeText("* other\n")

            val fs = JvmFileSystem(basePath = root)
            val files = fs.listOrgFiles(root.pathString, recursive = true, IgnorePatterns.EMPTY).toList()

            assertTrue("arroyo-system/template/inside.org should be excluded") {
                files.none { it.endsWith("arroyo-system/template/inside.org") }
            }
            assertTrue("keep.org must remain") { files.any { it.endsWith("keep.org") } }
            assertTrue("a.org must remain") { files.any { it.endsWith("a.org") } }
            // Bare 'template' is anchored to arroyo-system/, so the deeper
            // template/ dir under sub/ is NOT excluded.
            assertTrue("arroyo-system/sub/template/inside.org must remain (anchored)") {
                files.any { it.endsWith("arroyo-system/sub/template/inside.org") }
            }
            assertTrue("other/template/inside.org must remain (out of scope)") {
                files.any { it.endsWith("other/template/inside.org") }
            }
        } finally {
            root.toFile().deleteRecursively()
        }
    }

    @Test
    fun `nested arcologyignore with double-star excludes deeper descendants`() = runBlocking {
        val root = createTempDirectory(prefix = "arcology-test-")
        try {
            root.resolve("a.org").writeText("* A\n")
            val sub = root.resolve("arroyo-system").createDirectories()
            sub.resolve(".arcologyignore").writeText("**/template\n")
            sub.resolve("template").createDirectories().resolve("inside.org").writeText("* inside\n")
            sub.resolve("keep.org").writeText("* keep\n")
            sub.resolve("x").createDirectories().resolve("template").createDirectories().resolve("inside.org").writeText("* deep\n")

            val fs = JvmFileSystem(basePath = root)
            val files = fs.listOrgFiles(root.pathString, recursive = true, IgnorePatterns.EMPTY).toList()

            assertTrue("arroyo-system/template/inside.org excluded") {
                files.none { it.endsWith("arroyo-system/template/inside.org") }
            }
            assertTrue("arroyo-system/x/template/inside.org excluded by **") {
                files.none { it.endsWith("arroyo-system/x/template/inside.org") }
            }
            assertTrue("keep.org remains") { files.any { it.endsWith("keep.org") } }
            assertTrue("a.org remains") { files.any { it.endsWith("a.org") } }
        } finally {
            root.toFile().deleteRecursively()
        }
    }

    @Test
    fun `nested arcologyignore accumulates on top of root ignore`() = runBlocking {
        val root = createTempDirectory(prefix = "arcology-test-")
        try {
            // Root '*.bak' excludes top-level .bak files. A nested
            // 'arroyo-system/.arcologyignore' with 'template' prunes the
            // 'template' directory within arroyo-system/.
            root.resolve(".arcologyignore").writeText("*.bak\n")
            root.resolve("notes.bak").writeText("bak\n")
            root.resolve("notes.org").writeText("* notes\n")
            val sub = root.resolve("arroyo-system").createDirectories()
            sub.resolve(".arcologyignore").writeText("template\n")
            sub.resolve("template").createDirectories().resolve("inside.org").writeText("* inside\n")
            sub.resolve("keep.org").writeText("* keep\n")

            val fs = JvmFileSystem(basePath = root)
            val rootPatterns = IgnorePatterns.parse(fs.readIgnoreFile(root.pathString)!!)
            val files = fs.listOrgFiles(root.pathString, recursive = true, rootPatterns).toList()

            assertTrue("notes.bak excluded by root scope") {
                files.none { it.endsWith("notes.bak") }
            }
            assertTrue("arroyo-system/template/inside.org excluded by nested scope") {
                files.none { it.endsWith("arroyo-system/template/inside.org") }
            }
            assertTrue("keep.org remains") { files.any { it.endsWith("keep.org") } }
            assertTrue("notes.org remains") { files.any { it.endsWith("notes.org") } }
        } finally {
            root.toFile().deleteRecursively()
        }
    }
}