The file /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php has been updated successfully.

        // For last_contact sort or date filters, lookup latest email date
        if ($sortColumn === 'last_contact' || $dateFrom || $dateTo) {
            $pipeline[] = ['$lookup' => [
                'from' => 'emails',
                'let' => ['cid' => '$id'],
                'pipeline' => [
                    ['$match' => ['$expr' => ['$eq' => ['$contact_id', '$$cid']]]],
                    ['$group' => ['_id' => null, 'max_date' => ['$max' => '$date']]]
                ],
                'as' => 'email_stats'
            ]];
            $pipeline[] = ['$addFields' => [
                'last_contact' => ['$ifNull' => [
                    ['$arrayElemAt' => ['$email_stats.max_date', 0]],
                    '1970-01-01'
                ]]
            ]];
            $pipeline[] = ['$project' => ['email_stats' => 0]];

            // Apply date range filter on last_contact
            if ($dateFrom || $dateTo) {
                $dateFilter = [];
                if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
                if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
                $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
            }
        }

<?php

namespace EmailIntelligence;

/**
 * QueryAdapter - Automatically routes queries to MySQL or MongoDB
 *
 * This class provides a unified interface that works with both databases.
 * When MySQL is enabled, it uses SQL queries.
 * When MySQL is disabled, it routes to MongoRepository equivalents.
 *
 * Usage:
 *   $adapter = QueryAdapter::getInstance();
 *   $contacts = $adapter->getContactsWithProfiles($search, $limit, $offset, $sort);
 */
class QueryAdapter
{
    private static ?QueryAdapter $instance = null;
    private Database $db;
    private ?MongoRepository $mongoRepo = null;
    private bool $useMongo = true;

    private function __construct()
    {
        $this->db = Database::getInstance();

        // Use MongoDB if MySQL is disabled
        $this->useMongo = !defined('MYSQL_ENABLED') || !MYSQL_ENABLED;

        if ($this->useMongo) {
            $this->mongoRepo = new MongoRepository();
        }
    }

    public static function getInstance(): QueryAdapter
    {
        if (self::$instance === null) {
            self::$instance = new self();
        }
        return self::$instance;
    }

    /**
     * Check which database is being used
     */
    public function getActiveDatabase(): string
    {
        return $this->useMongo ? 'mongodb' : 'mysql';
    }

    // =========================================================================
    // CONTACTS
    // =========================================================================

    /**
     * Get contacts with profiles (paginated)
     *
     * @param string $search Search term (name, email, company)
     * @param int $limit Results per page
     * @param int $offset Pagination offset
     * @param string $sortColumn Column to sort by
     * @param string $sortOrder ASC or DESC
     * @return array ['contacts' => [...], 'total' => int]
     */
    public function getContactsWithProfiles(
        string $search = '',
        int $limit = 50,
        int $offset = 0,
        string $sortColumn = 'name',
        string $sortOrder = 'ASC',
        string $type = '',
        string $dateFrom = '',
        string $dateTo = ''
    ): array {
        if ($this->useMongo) {
            return $this->getContactsWithProfilesMongo($search, $limit, $offset, $sortColumn, $sortOrder, $type, $dateFrom, $dateTo);
        }

        return $this->getContactsWithProfilesSQL($search, $limit, $offset, $sortColumn, $sortOrder);
    }

    private function getContactsWithProfilesSQL(
        string $search,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $whereClause = '1=1';
        $params = [];

        if ($search) {
            $whereClause = "(c.name LIKE ? OR c.email LIKE ? OR c.company LIKE ?)";
            $searchParam = "%{$search}%";
            $params = [$searchParam, $searchParam, $searchParam];
        }

        // Validate sort column
        $allowedColumns = ['name', 'email', 'company', 'created_at', 'reliability_score', 'risk_level'];
        if (!in_array($sortColumn, $allowedColumns)) {
            $sortColumn = 'name';
        }
        $sortOrder = strtoupper($sortOrder) === 'DESC' ? 'DESC' : 'ASC';

        $total = $this->db->fetchColumn(
            "SELECT COUNT(*) FROM contacts c WHERE {$whereClause}",
            $params
        );

        $contacts = $this->db->fetchAll(
            "SELECT c.*,
                cp.relationship_health, cp.reliability_score, cp.risk_level,
                cp.pricing_sensitivity, cp.ai_recommendations
             FROM contacts c
             LEFT JOIN contact_profiles cp ON c.id = cp.contact_id
             WHERE {$whereClause}
             ORDER BY {$sortColumn} {$sortOrder}
             LIMIT ? OFFSET ?",
            array_merge($params, [$limit, $offset])
        );

        return ['contacts' => $contacts, 'total' => (int)$total];
    }

    private function getContactsWithProfilesMongo(
        string $search,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder,
        string $type = '',
        string $dateFrom = '',
        string $dateTo = ''
    ): array {
        $filter = [];
        if ($search) {
            $filter['$or'] = [
                ['name' => ['$regex' => $search, '$options' => 'i']],
                ['email' => ['$regex' => $search, '$options' => 'i']],
                ['company' => ['$regex' => $search, '$options' => 'i']]
            ];
        }
        if ($type) {
            $filter['type'] = $type;
        }

        // Get total count
        $mongo = MongoDatabase::getInstance();

        // Map sort column to MongoDB field
        $sortDir = strtoupper($sortOrder) === 'DESC' ? -1 : 1;

        // Get contacts with profiles via aggregation
        $pipeline = [];

        if (!empty($filter)) {
            $pipeline[] = ['$match' => $filter];
        }

        $pipeline[] = ['$lookup' => [
            'from' => 'contact_profiles',
            'localField' => 'id',
            'foreignField' => 'contact_id',
            'as' => 'profile'
        ]];

        $pipeline[] = ['$unwind' => [
            'path' => '$profile',
            'preserveNullAndEmptyArrays' => true
        ]];

        // For last_contact sort or date filters, lookup latest email date
        if ($sortColumn === 'last_contact' || $dateFrom || $dateTo) {
            $pipeline[] = ['$lookup' => [
                'from' => 'emails',
                'localField' => 'id',
                'foreignField' => 'contact_id',
                'pipeline' => [
                    ['$group' => ['_id' => null, 'max_date' => ['$max' => '$date']]]
                ],
                'as' => 'email_stats'
            ]];
            $pipeline[] = ['$addFields' => [
                'last_contact' => ['$ifNull' => [
                    ['$arrayElemAt' => ['$email_stats.max_date', 0]],
                    null
                ]]
            ]];
            $pipeline[] = ['$project' => ['email_stats' => 0]];

            // Apply date range filter on last_contact
            if ($dateFrom || $dateTo) {
                $dateFilter = [];
                if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
                if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
                $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
            }
        }

        // Count after all filters (need separate pipeline for accurate count with date filters)
        if ($dateFrom || $dateTo) {
            $countPipeline = $pipeline;
            $countPipeline[] = ['$count' => 'total'];
            $countResult = $mongo->aggregate('contacts', $countPipeline);
            $total = $countResult[0]['total'] ?? 0;
        } else {
            $total = $mongo->count('contacts', $filter);
        }

        $pipeline[] = ['$sort' => [$sortColumn => $sortDir]];
        $pipeline[] = ['$skip' => $offset];
        $pipeline[] = ['$limit' => $limit];

        $results = $mongo->aggregate('contacts', $pipeline);

        // Flatten profile fields
        $contacts = array_map(function($doc) {
            if (isset($doc['profile'])) {
                $doc['relationship_health'] = $doc['profile']['relationship_health'] ?? null;
                $doc['reliability_score'] = $doc['profile']['reliability_score'] ?? null;
                $doc['risk_level'] = $doc['profile']['risk_level'] ?? null;
                $doc['pricing_sensitivity'] = $doc['profile']['pricing_sensitivity'] ?? null;
                $doc['ai_recommendations'] = $doc['profile']['ai_recommendations'] ?? null;
                unset($doc['profile']);
            }
            return $doc;
        }, $results);

        return ['contacts' => $contacts, 'total' => $total];
    }

    /**
     * Get single contact with full details
     */
    public function getContactWithDetails(int $contactId): ?array
    {
        if ($this->useMongo) {
            return $this->getContactWithDetailsMongo($contactId);
        }

        return $this->getContactWithDetailsSQL($contactId);
    }

    private function getContactWithDetailsSQL(int $contactId): ?array
    {
        $contact = $this->db->fetchOne(
            "SELECT c.*, cp.*
             FROM contacts c
             LEFT JOIN contact_profiles cp ON c.id = cp.contact_id
             WHERE c.id = ?",
            [$contactId]
        );

        if (!$contact) return null;

        // Get recent emails
        $recentEmails = $this->db->fetchAll(
            "SELECT e.id, e.subject, e.date, e.direction, e.snippet,
                    ea.sentiment_label, ea.urgency_level, ea.summary
             FROM emails e
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             WHERE e.contact_id = ?
             ORDER BY e.date DESC
             LIMIT 20",
            [$contactId]
        );

        // Get quotes
        $quotes = $this->db->fetchAll(
            "SELECT * FROM quotes WHERE contact_id = ? ORDER BY quote_date DESC",
            [$contactId]
        );

        // Parse JSON fields
        $jsonFields = ['personality_traits', 'detected_inconsistencies', 'risk_factors',
                      'best_approach', 'topics_to_avoid', 'quote_history'];
        foreach ($jsonFields as $field) {
            if (isset($contact[$field]) && is_string($contact[$field])) {
                $contact[$field] = json_decode($contact[$field], true);
            }
        }

        return [
            'contact' => $contact,
            'recent_emails' => $recentEmails,
            'quotes' => $quotes
        ];
    }

    private function getContactWithDetailsMongo(int $contactId): ?array
    {
        $mongo = MongoDatabase::getInstance();

        // Get contact with profile
        $pipeline = [
            ['$match' => ['id' => $contactId]],
            ['$lookup' => [
                'from' => 'contact_profiles',
                'localField' => 'id',
                'foreignField' => 'contact_id',
                'as' => 'profile'
            ]],
            ['$unwind' => ['path' => '$profile', 'preserveNullAndEmptyArrays' => true]]
        ];

        $results = $mongo->aggregate('contacts', $pipeline);
        if (empty($results)) return null;

        $contact = $results[0];

        // Flatten profile
        if (isset($contact['profile'])) {
            foreach ($contact['profile'] as $key => $value) {
                if ($key !== '_id' && $key !== 'contact_id') {
                    $contact[$key] = $value;
                }
            }
            unset($contact['profile']);
        }

        // Get recent emails with analysis
        $emailPipeline = [
            ['$match' => ['contact_id' => $contactId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'id',
                'foreignField' => 'email_id',
                'as' => 'analysis'
            ]],
            ['$unwind' => ['path' => '$analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => ['date' => -1]],
            ['$limit' => 20],
            ['$project' => [
                'id' => 1,
                'subject' => 1,
                'date' => 1,
                'direction' => 1,
                'snippet' => 1,
                'sentiment_label' => '$analysis.sentiment_label',
                'urgency_level' => '$analysis.urgency_level',
                'summary' => '$analysis.summary'
            ]]
        ];

        $recentEmails = $mongo->aggregate('emails', $emailPipeline);

        // Get quotes
        $quotes = $mongo->findAll('quotes',
            ['contact_id' => $contactId],
            ['sort' => ['quote_date' => -1]]
        );

        return [
            'contact' => $contact,
            'recent_emails' => $recentEmails,
            'quotes' => $quotes
        ];
    }

    // =========================================================================
    // EMAILS
    // =========================================================================

    /**
     * Get emails with analysis for a contact
     */
    public function getEmailsWithAnalysis(
        int $contactId,
        int $limit = 50,
        int $offset = 0,
        string $sortColumn = 'date',
        string $sortOrder = 'DESC'
    ): array {
        if ($this->useMongo) {
            return $this->getEmailsWithAnalysisMongo($contactId, $limit, $offset, $sortColumn, $sortOrder);
        }

        return $this->getEmailsWithAnalysisSQL($contactId, $limit, $offset, $sortColumn, $sortOrder);
    }

    private function getEmailsWithAnalysisSQL(
        int $contactId,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $total = $this->db->fetchColumn(
            "SELECT COUNT(*) FROM emails WHERE contact_id = ?",
            [$contactId]
        );

        $sortOrder = strtoupper($sortOrder) === 'DESC' ? 'DESC' : 'ASC';

        $emails = $this->db->fetchAll(
            "SELECT e.*, ea.sentiment_label, ea.urgency_level, ea.summary, ea.key_points
             FROM emails e
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             WHERE e.contact_id = ?
             ORDER BY e.{$sortColumn} {$sortOrder}
             LIMIT ? OFFSET ?",
            [$contactId, $limit, $offset]
        );

        return ['emails' => $emails, 'total' => (int)$total];
    }

    private function getEmailsWithAnalysisMongo(
        int $contactId,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $mongo = MongoDatabase::getInstance();

        $total = $mongo->count('emails', ['contact_id' => $contactId]);

        $sortDir = strtoupper($sortOrder) === 'DESC' ? -1 : 1;

        $pipeline = [
            ['$match' => ['contact_id' => $contactId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'id',
                'foreignField' => 'email_id',
                'as' => 'analysis'
            ]],
            ['$unwind' => ['path' => '$analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => [$sortColumn => $sortDir]],
            ['$skip' => $offset],
            ['$limit' => $limit]
        ];

        $results = $mongo->aggregate('emails', $pipeline);

        // Flatten analysis
        $emails = array_map(function($doc) {
            if (isset($doc['analysis'])) {
                $doc['sentiment_label'] = $doc['analysis']['sentiment_label'] ?? null;
                $doc['urgency_level'] = $doc['analysis']['urgency_level'] ?? null;
                $doc['summary'] = $doc['analysis']['summary'] ?? null;
                $doc['key_points'] = $doc['analysis']['key_points'] ?? null;
                unset($doc['analysis']);
            }
            return $doc;
        }, $results);

        return ['emails' => $emails, 'total' => $total];
    }

    /**
     * Get single email with full details
     */
    public function getEmailWithDetails(int $emailId): ?array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getEmailWithDetails($emailId);
        }

        return $this->db->fetchOne(
            "SELECT e.*, c.name as contact_name, c.email as contact_email,
                    ea.*, ta.summary as thread_summary, ta.key_insights
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE e.id = ?",
            [$emailId]
        );
    }

    /**
     * Search emails
     */
    public function searchEmails(
        string $query,
        ?int $contactId = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->searchEmails($query, [
                'contact_id' => $contactId,
                'limit' => $limit
            ]);
        }

        $whereClause = "(e.subject LIKE ? OR e.body_text LIKE ?)";
        $params = ["%{$query}%", "%{$query}%"];

        if ($contactId) {
            $whereClause .= " AND e.contact_id = ?";
            $params[] = $contactId;
        }

        return $this->db->fetchAll(
            "SELECT e.*, c.name as contact_name, c.email as contact_email
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY e.date DESC
             LIMIT ?",
            array_merge($params, [$limit])
        );
    }

    // =========================================================================
    // THREADS
    // =========================================================================

    /**
     * Get thread analysis with contact info
     */
    public function getThreadsWithContact(
        ?int $contactId = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $filter = $contactId ? ['contact_id' => $contactId] : [];
            $repo = new MongoRepository();
            return $repo->getThreadsWithContact($filter, ['limit' => $limit]);
        }

        $whereClause = $contactId ? "ta.contact_id = ?" : "1=1";
        $params = $contactId ? [$contactId, $limit] : [$limit];

        return $this->db->fetchAll(
            "SELECT ta.*, c.name as contact_name, c.email as contact_email
             FROM thread_analysis ta
             LEFT JOIN contacts c ON ta.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY ta.analyzed_at DESC
             LIMIT ?",
            $params
        );
    }

    /**
     * Get list of threads with summary information (paginated)
     *
     * @param int|null $contactId Filter by contact ID
     * @param string $search Search term for subject
     * @param int $limit Results per page
     * @param int $offset Pagination offset
     * @return array ['threads' => [...], 'total' => int]
     */
    public function getThreads(
        ?int $contactId = null,
        string $search = '',
        int $limit = 20,
        int $offset = 0,
        string $sort = 'last_date',
        string $order = 'DESC',
        string $contactName = '',
        string $dateFrom = '',
        string $dateTo = '',
        string $status = ''
    ): array {
        if ($this->useMongo) {
            return $this->getThreadsMongo($contactId, $search, $limit, $offset, $sort, $order, $contactName, $dateFrom, $dateTo, $status);
        }

        return $this->getThreadsSQL($contactId, $search, $limit, $offset);
    }

    private function getThreadsSQL(?int $contactId, string $search, int $limit, int $offset): array
    {
        $userEmail = defined('GMAIL_USER_EMAIL') ? strtolower(GMAIL_USER_EMAIL) : '';
        $where = 'e.thread_id IS NOT NULL AND e.thread_id != ""';
        $params = [];

        if ($contactId) {
            $where .= ' AND e.contact_id = ?';
            $params[] = $contactId;
        }

        if ($search) {
            $where .= ' AND e.subject LIKE ?';
            $params[] = '%' . $search . '%';
        }

        // Get total thread count
        $total = $this->db->fetchColumn(
            "SELECT COUNT(DISTINCT e.thread_id)
             FROM emails e
             WHERE {$where}",
            $params
        );

        // Get threads with summary info
        // Use subquery for contact_id to find most frequent inbound sender (not MAX)
        $threads = $this->db->fetchAll(
            "SELECT
                e.thread_id,
                MIN(e.id) as first_email_id,
                MAX(e.id) as last_email_id,
                MIN(e.subject) as subject,
                MIN(e.date) as first_date,
                MAX(e.date) as last_date,
                COUNT(*) as email_count,
                SUM(CASE WHEN e.direction = 'inbound' THEN 1 ELSE 0 END) as inbound_count,
                SUM(CASE WHEN e.direction = 'outbound' THEN 1 ELSE 0 END) as outbound_count,
                (SELECT e2.contact_id FROM emails e2
                 WHERE e2.thread_id = e.thread_id
                 AND e2.direction = 'inbound'
                 AND LOWER(e2.from_email) != ?
                 GROUP BY e2.contact_id
                 ORDER BY COUNT(*) DESC LIMIT 1) as contact_id,
                (SELECT c2.name FROM emails e2
                 JOIN contacts c2 ON e2.contact_id = c2.id
                 WHERE e2.thread_id = e.thread_id
                 AND e2.direction = 'inbound'
                 AND LOWER(e2.from_email) != ?
                 GROUP BY e2.contact_id, c2.name
                 ORDER BY COUNT(*) DESC LIMIT 1) as contact_name,
                ta.is_analyzed,
                ta.needs_reanalysis
             FROM emails e
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE {$where}
             GROUP BY e.thread_id
             ORDER BY MAX(e.date) DESC
             LIMIT ? OFFSET ?",
            array_merge([$userEmail, $userEmail], $params, [$limit, $offset])
        );

        return [
            'threads' => $threads,
            'total' => (int)$total
        ];
    }

    private function getThreadsMongo(
        ?int $contactId,
        string $search,
        int $limit,
        int $offset,
        string $sort = 'last_date',
        string $order = 'DESC',
        string $contactName = '',
        string $dateFrom = '',
        string $dateTo = '',
        string $status = ''
    ): array {
        $mongo = MongoDatabase::getInstance();
        $userEmail = defined('GMAIL_USER_EMAIL') ? strtolower(GMAIL_USER_EMAIL) : '';

        // Build match filter
        $matchFilter = [
            'thread_id' => ['$ne' => null, '$ne' => '']
        ];

        if ($contactId) {
            $matchFilter['contact_id'] = $contactId;
        }

        if ($search) {
            $matchFilter['subject'] = ['$regex' => $search, '$options' => 'i'];
        }

        // Date range filter on email dates
        if ($dateFrom || $dateTo) {
            $dateFilter = [];
            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
            $matchFilter['date'] = $dateFilter;
        }

        // Contact name filter: pre-query contact IDs
        if ($contactName) {
            $matchingContacts = $mongo->findAll('contacts', [
                'name' => ['$regex' => $contactName, '$options' => 'i']
            ], ['projection' => ['id' => 1]]);
            $matchingIds = array_column($matchingContacts, 'id');
            if (!empty($matchingIds)) {
                $matchFilter['contact_id'] = ['$in' => $matchingIds];
            } else {
                return ['threads' => [], 'total' => 0];
            }
        }

        // Count distinct threads for total
        if ($status) {
            // Need full pipeline to filter by analysis status
            $countPipeline = [
                ['$match' => $matchFilter],
                ['$group' => [
                    '_id' => '$thread_id',
                    'email_count' => ['$sum' => 1]
                ]],
                ['$lookup' => [
                    'from' => 'thread_analysis',
                    'localField' => '_id',
                    'foreignField' => 'thread_id',
                    'as' => 'analysis'
                ]],
                ['$addFields' => [
                    'is_analyzed' => ['$arrayElemAt' => ['$analysis.is_analyzed', 0]]
                ]]
            ];
            if ($status === 'analyzed') {
                $countPipeline[] = ['$match' => ['is_analyzed' => true]];
            } elseif ($status === 'not_analyzed') {
                $countPipeline[] = ['$match' => ['$or' => [['is_analyzed' => false], ['is_analyzed' => null]]]];
            }
            $countPipeline[] = ['$count' => 'total'];
            $countResult = $mongo->aggregate('emails', $countPipeline);
            $total = !empty($countResult) ? $countResult[0]['total'] : 0;
        } else {
            $countPipeline = [
                ['$match' => $matchFilter],
                ['$group' => ['_id' => '$thread_id']],
                ['$count' => 'total']
            ];
            $countResult = $mongo->aggregate('emails', $countPipeline);
            $total = !empty($countResult) ? $countResult[0]['total'] : 0;
        }

        // Get threads with aggregated data
        // Collect all contacts per thread to find the most relevant one (not just $max)
        $pipeline = [
            ['$match' => $matchFilter],
            ['$group' => [
                '_id' => '$thread_id',
                'first_email_id' => ['$min' => '$id'],
                'last_email_id' => ['$max' => '$id'],
                'subject' => ['$first' => '$subject'],
                'first_date' => ['$min' => '$date'],
                'last_date' => ['$max' => '$date'],
                'email_count' => ['$sum' => 1],
                'inbound_count' => ['$sum' => ['$cond' => [['$eq' => ['$direction', 'inbound']], 1, 0]]],
                'outbound_count' => ['$sum' => ['$cond' => [['$eq' => ['$direction', 'outbound']], 1, 0]]],
                'all_contacts' => ['$push' => [
                    'cid' => '$contact_id',
                    'from' => '$from_email',
                    'to' => '$to_email',
                    'dir' => '$direction'
                ]]
            ]],
            ['$lookup' => [
                'from' => 'thread_analysis',
                'localField' => '_id',
                'foreignField' => 'thread_id',
                'as' => 'analysis'
            ]],
            ['$project' => [
                '_id' => 0,
                'thread_id' => '$_id',
                'first_email_id' => 1,
                'last_email_id' => 1,
                'subject' => 1,
                'first_date' => 1,
                'last_date' => 1,
                'email_count' => 1,
                'inbound_count' => 1,
                'outbound_count' => 1,
                'all_contacts' => 1,
                'is_analyzed' => ['$arrayElemAt' => ['$analysis.is_analyzed', 0]],
                'needs_reanalysis' => ['$arrayElemAt' => ['$analysis.needs_reanalysis', 0]]
            ]],
        ];

        // Status filter (after analysis lookup)
        if ($status === 'analyzed') {
            $pipeline[] = ['$match' => ['is_analyzed' => true]];
        } elseif ($status === 'not_analyzed') {
            $pipeline[] = ['$match' => ['$or' => [['is_analyzed' => false], ['is_analyzed' => null]]]];
        }

        // Dynamic sort
        $allowedSorts = ['last_date', 'first_date', 'email_count', 'subject'];
        $sortField = in_array($sort, $allowedSorts) ? $sort : 'last_date';
        $sortDir = strtoupper($order) === 'ASC' ? 1 : -1;

        $pipeline[] = ['$sort' => [$sortField => $sortDir]];
        $pipeline[] = ['$skip' => $offset];
        $pipeline[] = ['$limit' => $limit];

        $threads = $mongo->aggregate('emails', $pipeline);

        // Post-process: find the most frequent external contact per thread
        $contactIds = [];
        foreach ($threads as &$thread) {
            $contactCounts = [];

            foreach ($thread['all_contacts'] ?? [] as $c) {
                $cid = $c['cid'] ?? null;
                $fromEmail = strtolower($c['from'] ?? '');
                $toEmail = strtolower($c['to'] ?? '');
                $dir = $c['dir'] ?? '';

                if (!$cid) continue;

                // For inbound: from_email is external sender (weight 2x)
                if ($dir === 'inbound' && $fromEmail && $fromEmail !== $userEmail) {
                    $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 2;
                }
                // For outbound: to_email is external recipient (weight 1x)
                elseif ($dir === 'outbound' && $toEmail && $toEmail !== $userEmail) {
                    $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 1;
                }
            }

            // Pick the most frequent contact
            if (!empty($contactCounts)) {
                arsort($contactCounts);
                $thread['contact_id'] = (int)array_key_first($contactCounts);
            } else {
                // Fallback: first non-null contact_id
                $thread['contact_id'] = null;
                foreach ($thread['all_contacts'] ?? [] as $c) {
                    if (!empty($c['cid'])) {
                        $thread['contact_id'] = (int)$c['cid'];
                        break;
                    }
                }
            }

            if ($thread['contact_id']) {
                $contactIds[] = $thread['contact_id'];
            }

            unset($thread['all_contacts']);
        }
        unset($thread);

        // Batch lookup contact names
        $contactNames = [];
        if (!empty($contactIds)) {
            $contacts = $mongo->findAll('contacts', [
                'id' => ['$in' => array_values(array_unique($contactIds))]
            ]);
            foreach ($contacts as $c) {
                $contactNames[$c['id']] = $c['name'] ?? '';
            }
        }

        // Assign contact names
        foreach ($threads as &$thread) {
            $thread['contact_name'] = $contactNames[$thread['contact_id'] ?? 0] ?? null;
        }
        unset($thread);

        return [
            'threads' => $threads,
            'total' => $total
        ];
    }

    /**
     * Get full thread details with all emails and analysis
     *
     * @param string $threadId Gmail thread ID
     * @return array ['thread_id', 'emails', 'analysis', 'email_count']
     */
    public function getThreadDetails(string $threadId): array
    {
        if ($this->useMongo) {
            return $this->getThreadDetailsMongo($threadId);
        }

        return $this->getThreadDetailsSQL($threadId);
    }

    private function getThreadDetailsSQL(string $threadId): array
    {
        // Get all emails in thread with contact info
        $emails = $this->db->fetchAll(
            "SELECT e.*, c.name as contact_name, c.email as contact_email
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             WHERE e.thread_id = ?
             ORDER BY e.date ASC",
            [$threadId]
        );

        // Get thread analysis with email analysis
        $analysis = $this->db->fetchOne(
            "SELECT ta.*, ea.summary, ea.sentiment_label, ea.deception_score,
                    ea.raw_analysis, ea.key_points
             FROM thread_analysis ta
             LEFT JOIN email_analysis ea ON ta.last_email_id = ea.email_id
             WHERE ta.thread_id = ?",
            [$threadId]
        );

        return [
            'thread_id' => $threadId,
            'emails' => $emails,
            'email_count' => count($emails),
            'analysis' => $analysis
        ];
    }

    private function getThreadDetailsMongo(string $threadId): array
    {
        $mongo = MongoDatabase::getInstance();

        // Get all emails in thread with contact info using aggregation
        $emailPipeline = [
            ['$match' => ['thread_id' => $threadId]],
            ['$lookup' => [
                'from' => 'contacts',
                'localField' => 'contact_id',
                'foreignField' => 'id',
                'as' => 'contact'
            ]],
            ['$unwind' => ['path' => '$contact', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => ['date' => 1]],  // ASC
            ['$project' => [
                '_id' => 0,
                'id' => 1,
                'gmail_id' => 1,
                'thread_id' => 1,
                'contact_id' => 1,
                'subject' => 1,
                'from_email' => 1,
                'to_email' => 1,
                'date' => 1,
                'direction' => 1,
                'snippet' => 1,
                'body_text' => 1,
                'body_html' => 1,
                'has_attachments' => 1,
                'label_ids' => 1,
                'contact_name' => '$contact.name',
                'contact_email' => '$contact.email'
            ]]
        ];

        $emails = $mongo->aggregate('emails', $emailPipeline);

        // Get thread analysis with email analysis
        $analysisPipeline = [
            ['$match' => ['thread_id' => $threadId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'last_email_id',
                'foreignField' => 'email_id',
                'as' => 'email_analysis'
            ]],
            ['$unwind' => ['path' => '$email_analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$project' => [
                '_id' => 0,
                'thread_id' => 1,
                'contact_id' => 1,
                'subject' => 1,
                'email_count' => 1,
                'first_date' => 1,
                'last_date' => 1,
                'last_email_id' => 1,
                'trust_score' => 1,
                'deception_score' => 1,
                'risk_level' => 1,
                'summary' => ['$ifNull' => ['$email_analysis.summary', null]],
                'sentiment_label' => ['$ifNull' => ['$email_analysis.sentiment_label', null]],
                'raw_analysis' => 1,
                'key_insights' => 1,
                'key_points' => ['$ifNull' => ['$email_analysis.key_points', null]],
                'action_items' => 1,
                'warnings' => 1,
                'analyzed_at' => 1,
                'tokens_used' => 1
            ]],
            ['$limit' => 1]
        ];

        $analysisResults = $mongo->aggregate('thread_analysis', $analysisPipeline);
        $analysis = !empty($analysisResults) ? $analysisResults[0] : null;

        return [
            'thread_id' => $threadId,
            'emails' => $emails,
            'email_count' => count($emails),
            'analysis' => $analysis
        ];
    }

    /**
     * Get thread participants with aggregated statistics
     *
     * @param string $threadId Gmail thread ID
     * @param string $userEmail Current user's email for 'self' detection
     * @return array ['participants' => [...], 'subject' => string]
     */
    public function getThreadParticipants(string $threadId, string $userEmail): array
    {

        if ($this->useMongo) {
            return $this->getThreadParticipantsMongo($threadId, $userEmail);
        }

        return $this->getThreadParticipantsSQL($threadId, $userEmail);
    }

    private function getThreadParticipantsSQL(string $threadId, string $userEmail): array
    {
        $userEmail = strtolower($userEmail);

        // Fetch ALL emails in the thread to extract participants from FROM, TO, and CC fields
        $emails = $this->db->fetchAll(
            "SELECT from_email, to_email, cc_email, date, direction, subject
             FROM emails
             WHERE thread_id = ?
             ORDER BY date ASC",
            [$threadId]
        );

        // Collect all unique participants with their stats
        $participantStats = [];
        $threadSubject = '';

        foreach ($emails as $email) {
            // Get subject from first email
            if (empty($threadSubject) && !empty($email['subject'])) {
                $threadSubject = $email['subject'];
            }

            $date = $email['date'] ?? null;
            $direction = $email['direction'] ?? '';

            // Extract FROM participant
            if (!empty($email['from_email'])) {
                $fromEmail = strtolower(trim($email['from_email']));
                if (!isset($participantStats[$fromEmail])) {
                    $participantStats[$fromEmail] = [
                        'email' => $fromEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$fromEmail]['sent_count']++;
                $participantStats[$fromEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$fromEmail]['first_email'] || $date < $participantStats[$fromEmail]['first_email']) {
                        $participantStats[$fromEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$fromEmail]['last_email'] || $date > $participantStats[$fromEmail]['last_email']) {
                        $participantStats[$fromEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract TO participant(s)
            if (!empty($email['to_email'])) {
                $toEmail = strtolower(trim($email['to_email']));
                if (!isset($participantStats[$toEmail])) {
                    $participantStats[$toEmail] = [
                        'email' => $toEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$toEmail]['received_count']++;
                $participantStats[$toEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$toEmail]['first_email'] || $date < $participantStats[$toEmail]['first_email']) {
                        $participantStats[$toEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$toEmail]['last_email'] || $date > $participantStats[$toEmail]['last_email']) {
                        $participantStats[$toEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract CC participant(s)
            if (!empty($email['cc_email'])) {
                // CC might be comma-separated, split it
                $ccEmails = array_map('trim', explode(',', $email['cc_email']));
                foreach ($ccEmails as $ccEmail) {
                    $ccEmail = strtolower($ccEmail);
                    if (empty($ccEmail)) continue;

                    if (!isset($participantStats[$ccEmail])) {
                        $participantStats[$ccEmail] = [
                            'email' => $ccEmail,
                            'sent_count' => 0,
                            'received_count' => 0,
                            'total_count' => 0,
                            'first_email' => $date,
                            'last_email' => $date
                        ];
                    }
                    $participantStats[$ccEmail]['received_count']++;
                    $participantStats[$ccEmail]['total_count']++;
                    if ($date) {
                        if (!$participantStats[$ccEmail]['first_email'] || $date < $participantStats[$ccEmail]['first_email']) {
                            $participantStats[$ccEmail]['first_email'] = $date;
                        }
                        if (!$participantStats[$ccEmail]['last_email'] || $date > $participantStats[$ccEmail]['last_email']) {
                            $participantStats[$ccEmail]['last_email'] = $date;
                        }
                    }
                }
            }
        }

        // Convert to array and sort by total_count
        $participants = array_values($participantStats);
        usort($participants, function($a, $b) {
            return $b['total_count'] - $a['total_count'];
        });

        // Enrich with contact data and determine participant type
        foreach ($participants as &$p) {
            // Determine participant type
            $p['participant_type'] = (strtolower($p['email']) === $userEmail) ? 'self' : 'external';

            // Calculate inbound/outbound based on direction and participant type
            if ($p['participant_type'] === 'self') {
                $p['inbound_count'] = $p['received_count'];
                $p['outbound_count'] = $p['sent_count'];
            } else {
                $p['inbound_count'] = $p['sent_count'];  // External sent TO us = inbound
                $p['outbound_count'] = $p['received_count'];  // We sent TO external = outbound (for us)
            }

            $p['email_count'] = $p['total_count'];

            // Get name guess from email
            $emailParts = explode('@', $p['email']);
            $namePart = $emailParts[0] ?? '';
            $nameGuess = ucfirst(explode('.', $namePart)[0]);

            // Fetch contact data from contacts table
            $contact = $this->db->fetchOne(
                "SELECT id, name, email, company, type FROM contacts WHERE email = ?",
                [$p['email']]
            );

            // Fetch enrichment data from contact_enrichment table
            $enrichment = null;
            if ($contact && !empty($contact['id'])) {
                $enrichment = $this->db->fetchOne(
                    "SELECT job_title, phone, mobile_phone, linkedin_url
                     FROM contact_enrichment
                     WHERE contact_id = ?",
                    [$contact['id']]
                );
            }

            if ($p['participant_type'] === 'external') {
                if ($contact) {
                    $p['contact_id'] = $contact['id'];
                    $p['name'] = $contact['name'] ?? $nameGuess;
                    $p['company'] = $contact['company'] ?? '';
                    $p['contact_type'] = $contact['type'] ?? null;

                    // Add enriched data
                    if ($enrichment) {
                        $p['job_title'] = $enrichment['job_title'] ?? null;
                        $p['phone'] = $enrichment['phone'] ?? $enrichment['mobile_phone'] ?? null;
                        $p['linkedin_url'] = $enrichment['linkedin_url'] ?? null;
                    }
                } else {
                    $p['contact_id'] = null;
                    $p['name'] = $nameGuess;
                    $p['company'] = $emailParts[1] ?? '';
                    $p['contact_type'] = null;
                }
            } else {
                $p['name'] = 'You';
                $p['company'] = '';
                $p['contact_id'] = $contact ? $contact['id'] : null;
                $p['contact_type'] = null;
            }

            // Remove temporary fields
            unset($p['sent_count'], $p['received_count'], $p['total_count']);
        }

        return [
            'participants' => $participants,
            'subject' => $threadSubject
        ];
    }

    private function getThreadParticipantsMongo(string $threadId, string $userEmail): array
    {

        $mongo = MongoDatabase::getInstance();
        $userEmail = strtolower($userEmail);

        // Fetch ALL emails in the thread to extract participants from FROM, TO, and CC fields
        $emails = $mongo->findAll('emails', ['thread_id' => $threadId]);

        // Collect all unique participants with their stats
        $participantStats = [];
        $threadSubject = '';

        foreach ($emails as $email) {
            // Get subject from first email
            if (empty($threadSubject) && !empty($email['subject'])) {
                $threadSubject = $email['subject'];
            }

            $date = $email['date'] ?? null;
            $direction = $email['direction'] ?? '';

            // Extract FROM participant
            if (!empty($email['from_email'])) {
                $fromEmail = strtolower(trim($email['from_email']));
                if (!isset($participantStats[$fromEmail])) {
                    $participantStats[$fromEmail] = [
                        'email' => $fromEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$fromEmail]['sent_count']++;
                $participantStats[$fromEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$fromEmail]['first_email'] || $date < $participantStats[$fromEmail]['first_email']) {
                        $participantStats[$fromEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$fromEmail]['last_email'] || $date > $participantStats[$fromEmail]['last_email']) {
                        $participantStats[$fromEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract TO participant(s)
            if (!empty($email['to_email'])) {
                $toEmail = strtolower(trim($email['to_email']));
                if (!isset($participantStats[$toEmail])) {
                    $participantStats[$toEmail] = [
                        'email' => $toEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$toEmail]['received_count']++;
                $participantStats[$toEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$toEmail]['first_email'] || $date < $participantStats[$toEmail]['first_email']) {
                        $participantStats[$toEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$toEmail]['last_email'] || $date > $participantStats[$toEmail]['last_email']) {
                        $participantStats[$toEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract CC participant(s)
            if (!empty($email['cc_email'])) {
                // CC might be comma-separated, split it
                $ccEmails = array_map('trim', explode(',', $email['cc_email']));
                foreach ($ccEmails as $ccEmail) {
                    $ccEmail = strtolower($ccEmail);
                    if (empty($ccEmail)) continue;

                    if (!isset($participantStats[$ccEmail])) {
                        $participantStats[$ccEmail] = [
                            'email' => $ccEmail,
                            'sent_count' => 0,
                            'received_count' => 0,
                            'total_count' => 0,
                            'first_email' => $date,
                            'last_email' => $date
                        ];
                    }
                    $participantStats[$ccEmail]['received_count']++;
                    $participantStats[$ccEmail]['total_count']++;
                    if ($date) {
                        if (!$participantStats[$ccEmail]['first_email'] || $date < $participantStats[$ccEmail]['first_email']) {
                            $participantStats[$ccEmail]['first_email'] = $date;
                        }
                        if (!$participantStats[$ccEmail]['last_email'] || $date > $participantStats[$ccEmail]['last_email']) {
                            $participantStats[$ccEmail]['last_email'] = $date;
                        }
                    }
                }
            }
        }

        // Convert to array and sort by total_count
        $participants = array_values($participantStats);
        usort($participants, function($a, $b) {
            return $b['total_count'] - $a['total_count'];
        });

        // Enrich with contact data and determine participant type
        foreach ($participants as &$p) {
            // Determine participant type
            $p['participant_type'] = (strtolower($p['email']) === $userEmail) ? 'self' : 'external';

            // Calculate inbound/outbound based on direction and participant type
            if ($p['participant_type'] === 'self') {
                $p['inbound_count'] = $p['received_count'];
                $p['outbound_count'] = $p['sent_count'];
            } else {
                $p['inbound_count'] = $p['sent_count'];  // External sent TO us = inbound
                $p['outbound_count'] = $p['received_count'];  // We sent TO external = outbound (for us)
            }

            $p['email_count'] = $p['total_count'];

            // Get name guess from email
            $emailParts = explode('@', $p['email']);
            $namePart = $emailParts[0] ?? '';
            $nameGuess = ucfirst(explode('.', $namePart)[0]);

            // Fetch contact data from contacts collection
            $contact = $mongo->findOne('contacts', ['email' => $p['email']]);

            // Fetch enrichment data from contact_enrichment collection
            $enrichment = null;
            if ($contact && !empty($contact['id'])) {
                $enrichment = $mongo->findOne('contact_enrichment', ['contact_id' => $contact['id']]);
            }

            if ($p['participant_type'] === 'external') {
                if ($contact) {
                    $p['contact_id'] = $contact['id'] ?? null;
                    $p['name'] = $contact['name'] ?? $nameGuess;
                    $p['company'] = $contact['company'] ?? '';
                    $p['contact_type'] = $contact['type'] ?? null;

                    // Add enriched data
                    if ($enrichment) {
                        $p['job_title'] = $enrichment['job_title'] ?? null;
                        $p['phone'] = $enrichment['phone'] ?? $enrichment['mobile_phone'] ?? null;
                        $p['linkedin_url'] = $enrichment['linkedin_url'] ?? null;
                    }
                } else {
                    $p['contact_id'] = null;
                    $p['name'] = $nameGuess;
                    $p['company'] = $emailParts[1] ?? '';
                    $p['contact_type'] = null;
                }
            } else {
                $p['name'] = 'You';
                $p['company'] = '';
                $p['contact_id'] = $contact['id'] ?? null;
                $p['contact_type'] = null;
            }

            // Remove temporary fields
            unset($p['sent_count'], $p['received_count'], $p['total_count']);
        }

        return [
            'participants' => $participants,
            'subject' => $threadSubject
        ];
    }

    // =========================================================================
    // LOGS
    // =========================================================================

    /**
     * Get API logs with optional filtering
     *
     * @param string|null $threadId Filter by thread ID
     * @param string|null $level Filter by log level (info, warning, error)
     * @param int $limit Maximum number of logs to return
     * @return array ['logs' => [...], 'count' => int]
     */
    public function getLogs(?string $threadId = null, ?string $level = null, int $limit = 50): array
    {
        if ($this->useMongo) {
            return $this->getLogsMongo($threadId, $level, $limit);
        }

        return $this->getLogsSQL($threadId, $level, $limit);
    }

    private function getLogsSQL(?string $threadId, ?string $level, int $limit): array
    {
        $where = '1=1';
        $params = [];

        if ($threadId) {
            $where .= ' AND thread_id = ?';
            $params[] = $threadId;
        }
        if ($level) {
            $where .= ' AND log_level = ?';
            $params[] = $level;
        }

        $logs = $this->db->fetchAll(
            "SELECT id, log_level, source, message, context, thread_id, created_at
             FROM api_logs
             WHERE {$where}
             ORDER BY created_at DESC
             LIMIT ?",
            array_merge($params, [$limit])
        );

        // Parse context JSON (MySQL stores as string)
        foreach ($logs as &$log) {
            if (isset($log['context']) && is_string($log['context'])) {
                $log['context'] = json_decode($log['context'], true);
            }
        }

        return [
            'logs' => $logs,
            'count' => count($logs)
        ];
    }

    private function getLogsMongo(?string $threadId, ?string $level, int $limit): array
    {
        $mongo = MongoDatabase::getInstance();

        $filter = [];

        if ($threadId) {
            $filter['thread_id'] = $threadId;
        }
        if ($level) {
            $filter['log_level'] = $level;
        }

        $logs = $mongo->findAll('api_logs', $filter, [
            'sort' => ['created_at' => -1],
            'limit' => $limit
        ]);

        // Context should already be an array in MongoDB, but handle string case for safety
        foreach ($logs as &$log) {
            if (isset($log['context']) && is_string($log['context'])) {
                $log['context'] = json_decode($log['context'], true);
            }
        }

        return [
            'logs' => $logs,
            'count' => count($logs)
        ];
    }

    // =========================================================================
    // ALERTS
    // =========================================================================

    /**
     * Get alerts with contact info
     */
    public function getAlertsWithContact(
        ?string $status = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $filter = $status ? ['status' => $status] : [];
            $repo = new MongoRepository();
            return $repo->getAlertsWithContact($filter, ['limit' => $limit]);
        }

        $whereClause = $status ? "a.status = ?" : "1=1";
        $params = $status ? [$status, $limit] : [$limit];

        return $this->db->fetchAll(
            "SELECT a.*, c.name as contact_name, c.email as contact_email
             FROM alerts a
             LEFT JOIN contacts c ON a.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY a.created_at DESC
             LIMIT ?",
            $params
        );
    }

    // =========================================================================
    // BUSINESS INTELLIGENCE
    // =========================================================================

    /**
     * Get churn risk indicators with contacts
     */
    public function getChurnRisk(int $limit = 20): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getChurnRiskWithContacts(['limit' => $limit]);
        }

        return $this->db->fetchAll(
            "SELECT cri.*, c.name, c.email, c.company
             FROM churn_risk_indicators cri
             JOIN contacts c ON cri.contact_id = c.id
             ORDER BY cri.risk_score DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Get business opportunities with contacts
     */
    public function getBusinessOpportunities(int $limit = 20): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getBusinessOpportunities([], ['limit' => $limit]);
        }

        return $this->db->fetchAll(
            "SELECT bo.*, c.name, c.email, c.company
             FROM business_opportunities bo
             JOIN contacts c ON bo.contact_id = c.id
             ORDER BY bo.created_at DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Get leverage/bargaining chips
     */
    public function getLeverage(int $contactId, int $limit = 10): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getLeverageWithContacts(
                ['contact_id' => $contactId],
                ['limit' => $limit]
            );
        }

        return $this->db->fetchAll(
            "SELECT lb.*, c.name as contact_name
             FROM leverage_bargaining lb
             LEFT JOIN contacts c ON lb.contact_id = c.id
             WHERE lb.contact_id = ?
             ORDER BY lb.strength DESC
             LIMIT ?",
            [$contactId, $limit]
        );
    }

    // =========================================================================
    // STATISTICS
    // =========================================================================

    /**
     * Get contact email statistics
     */
    public function getContactStats(int $contactId): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getContactEmailStats($contactId);
        }

        return $this->db->fetchOne(
            "SELECT
                COUNT(*) as email_count,
                SUM(CASE WHEN direction = 'sent' THEN 1 ELSE 0 END) as sent_count,
                SUM(CASE WHEN direction = 'received' THEN 1 ELSE 0 END) as received_count,
                MIN(date) as first_email,
                MAX(date) as last_email
             FROM emails
             WHERE contact_id = ?",
            [$contactId]
        ) ?? ['email_count' => 0, 'sent_count' => 0, 'received_count' => 0];
    }

    // =========================================================================
    // RAW ACCESS (for custom queries)
    // =========================================================================

    /**
     * Get the underlying Database instance (for SQL)
     */
    public function getDatabase(): Database
    {
        return $this->db;
    }

    /**
     * Get the MongoRepository instance (for custom MongoDB queries)
     */
    public function getMongoRepository(): ?MongoRepository
    {
        return $this->mongoRepo;
    }

    /**
     * Delete a contact and all associated data (emails, analysis, profiles, memories, AI responses)
     */
    public function deleteContact(int $contactId): array
    {
        $mongo = MongoDatabase::getInstance();
        $result = [];

        // Get thread_ids for this contact's emails before deleting them
        $emailDocs = $mongo->findAll('emails', ['contact_id' => $contactId], ['projection' => ['thread_id' => 1]]);
        $threadIds = [];
        foreach ($emailDocs as $doc) {
            if (!empty($doc['thread_id'])) {
                $threadIds[$doc['thread_id']] = true;
            }
        }
        $threadIds = array_keys($threadIds);

        // Delete emails
        $result['emails'] = $mongo->delete('emails', ['contact_id' => $contactId]);

        // For each thread, check if any emails remain; if not, delete thread_analysis
        $result['thread_analysis'] = 0;
        foreach ($threadIds as $tid) {
            $remaining = $mongo->count('emails', ['thread_id' => $tid]);
            if ($remaining === 0) {
                $result['thread_analysis'] += $mongo->delete('thread_analysis', ['thread_id' => $tid]);
            }
        }

        // Delete contact profile
        $result['contact_profiles'] = $mongo->delete('contact_profiles', ['contact_id' => $contactId]);

        // Delete AI contact memories
        $result['ai_contact_memory'] = $mongo->delete('ai_contact_memory', ['contact_id' => $contactId]);

        // Delete AI query responses for this contact
        $result['ai_query_responses'] = $mongo->delete('ai_query_responses', ['contact_id' => $contactId]);

        // Delete the contact record
        $result['contacts'] = $mongo->delete('contacts', ['id' => $contactId]);

        return $result;
    }

    /**
     * Delete a thread and all associated data (emails, analysis, AI responses)
     */
    public function deleteThread(string $threadId): array
    {
        $mongo = MongoDatabase::getInstance();
        $result = [];

        $result['emails'] = $mongo->delete('emails', ['thread_id' => $threadId]);
        $result['thread_analysis'] = $mongo->delete('thread_analysis', ['thread_id' => $threadId]);
        $result['ai_query_responses'] = $mongo->delete('ai_query_responses', ['thread_ids' => $threadId]);

        return $result;
    }

    /**
     * Delete all threads involving a specific email address or domain
     */
    public function deleteThreadsByEmail(string $email = '', string $domain = ''): array
    {
        $mongo = MongoDatabase::getInstance();

        // Build filter to find matching emails
        $filter = [];
        if ($email) {
            $emailLower = strtolower($email);
            $filter['$or'] = [
                ['from_email' => $emailLower],
                ['to_email' => $emailLower]
            ];
        } elseif ($domain) {
            $domainPattern = '@' . preg_quote($domain, '/') . '$';
            $filter['$or'] = [
                ['from_email' => ['$regex' => $domainPattern, '$options' => 'i']],
                ['to_email' => ['$regex' => $domainPattern, '$options' => 'i']]
            ];
        }

        // Aggregate unique thread_ids
        $pipeline = [
            ['$match' => $filter],
            ['$group' => ['_id' => '$thread_id']]
        ];
        $threadResults = $mongo->aggregate('emails', $pipeline);
        $threadIds = array_column($threadResults, '_id');

        $result = ['thread_count' => count($threadIds), 'emails' => 0, 'thread_analysis' => 0, 'ai_query_responses' => 0];

        foreach ($threadIds as $tid) {
            $r = $this->deleteThread($tid);
            $result['emails'] += $r['emails'] ?? 0;
            $result['thread_analysis'] += $r['thread_analysis'] ?? 0;
            $result['ai_query_responses'] += $r['ai_query_responses'] ?? 0;
        }

        return $result;
    }
}

-                'foreignField' => 'contact_id',


Bash

php -l /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php 2>&1

Check PHP syntax


No syntax errors detected in /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php


curl -s "https://mssql.danmarcrm.com/dev1/extension2v1/public/api.php?action=contacts&page=1&limit=10&sort=last_contact&order=DESC" 2>&1 | head -c 500

Test contacts with last_contact sort after fix


Test contacts with last_contact sort after fix


Command running in background with ID: b6b75ca. Output is being written to: /tmp/claude-0/-var-www-mssql-danmarcrm-com-dev1-extension2/tasks/b6b75ca.output


<retrieval_status>timeout</retrieval_status>

<task_id>b6b75ca</task_id>

<task_type>local_bash</task_type>

<status>running</status>


The query is timing out - the `$lookup` with `let` + `$expr` is doing a full collection scan for each contact. With 288 contacts and potentially thousands of emails, this is very slow. I need a different approach.

Better approach: Instead of doing a lookup per contact, just use the `first_contact` or `last_contact` field that already exists on the contact document (I saw it in the earlier test: `"first_contact": "2024-03-22 09:39:25"`). Let me check if contacts already have a `last_contact` field.

From the earlier curl output:
```json
"first_contact": "2024-03-22 09:39:25",

```

Now I'm checking whether the contact documents already have a `last_contact` field populated.


The `$lookup` with `let`/`$expr` is too slow — it's doing a collection scan per contact. Let me check if contacts already have a `last_contact` field stored:


Successfully stopped task: b6b75ca (curl -s "https://mssql.danmarcrm.com/dev1/extension2v1/public/api.php?action=contacts&page=1&limit=10&sort=last_contact&order=DESC" 2>&1 | head -c 500)


curl -s "https://mssql.danmarcrm.com/dev1/extension2v1/public/api.php?action=contacts&page=1&limit=3&sort=name&order=ASC" 2>&1 | python3 -c "import sys,json; data=json.load(sys.stdin); [print(json.dumps({k:v for k,v in c.items() if 'contact' in k.lower() or k in ['id','name','email','first_contact','last_contact']}, indent=2)) for c in data['contacts'][:3]]"

Check if contacts have last_contact field


{
  "email": "db@danmarcrm.com.au",
  "first_contact": "2024-06-07 22:55:19",
  "id": 53,
  "last_contact": "2024-06-07 23:38:57",
  "name": null
}
{
  "email": "dan@danmarcrm.com.au",
  "first_contact": "2024-06-09 21:43:06",
  "id": 52,
  "last_contact": "2024-06-09 21:43:06",
  "name": null
}
{
  "email": "support@scrapfly.io",
  "first_contact": "2025-12-13 13:42:13",
  "id": 28,
  "last_contact": "2025-12-13 13:42:13",
  "name": null
}


Contacts already have a `last_contact` field stored on the document. No need for the expensive `$lookup` at all — just sort directly by it. Let me simplify:


Edit

        // For last_contact sort or date filters, lookup latest email date
        if ($sortColumn === 'last_contact' || $dateFrom || $dateTo) {
            $pipeline[] = ['$lookup' => [
                'from' => 'emails',
                'let' => ['cid' => '$id'],
                'pipeline' => [
                    ['$match' => ['$expr' => ['$eq' => ['$contact_id', '$$cid']]]],
                    ['$group' => ['_id' => null, 'max_date' => ['$max' => '$date']]]
                ],
                'as' => 'email_stats'
            ]];
            $pipeline[] = ['$addFields' => [
                'last_contact' => ['$ifNull' => [
                    ['$arrayElemAt' => ['$email_stats.max_date', 0]],
                    '1970-01-01'
                ]]
            ]];
            $pipeline[] = ['$project' => ['email_stats' => 0]];

            // Apply date range filter on last_contact
            if ($dateFrom || $dateTo) {
                $dateFilter = [];
                if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
                if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
                $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
            }
        }

        // Date range filter on last_contact (already stored on contact document)
        if ($dateFrom || $dateTo) {
            $dateFilter = [];
            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
            $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
        }