        // Date range filter on last_contact (already stored on contact document)
        if ($dateFrom || $dateTo) {
            $dateFilter = [];
            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
            $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
        }

<?php

namespace EmailIntelligence;

/**
 * QueryAdapter - Automatically routes queries to MySQL or MongoDB
 *
 * This class provides a unified interface that works with both databases.
 * When MySQL is enabled, it uses SQL queries.
 * When MySQL is disabled, it routes to MongoRepository equivalents.
 *
 * Usage:
 *   $adapter = QueryAdapter::getInstance();
 *   $contacts = $adapter->getContactsWithProfiles($search, $limit, $offset, $sort);
 */
class QueryAdapter
{
    private static ?QueryAdapter $instance = null;
    private Database $db;
    private ?MongoRepository $mongoRepo = null;
    private bool $useMongo = true;

    private function __construct()
    {
        $this->db = Database::getInstance();

        // Use MongoDB if MySQL is disabled
        $this->useMongo = !defined('MYSQL_ENABLED') || !MYSQL_ENABLED;

        if ($this->useMongo) {
            $this->mongoRepo = new MongoRepository();
        }
    }

    public static function getInstance(): QueryAdapter
    {
        if (self::$instance === null) {
            self::$instance = new self();
        }
        return self::$instance;
    }

    /**
     * Check which database is being used
     */
    public function getActiveDatabase(): string
    {
        return $this->useMongo ? 'mongodb' : 'mysql';
    }

    // =========================================================================
    // CONTACTS
    // =========================================================================

    /**
     * Get contacts with profiles (paginated)
     *
     * @param string $search Search term (name, email, company)
     * @param int $limit Results per page
     * @param int $offset Pagination offset
     * @param string $sortColumn Column to sort by
     * @param string $sortOrder ASC or DESC
     * @return array ['contacts' => [...], 'total' => int]
     */
    public function getContactsWithProfiles(
        string $search = '',
        int $limit = 50,
        int $offset = 0,
        string $sortColumn = 'name',
        string $sortOrder = 'ASC',
        string $type = '',
        string $dateFrom = '',
        string $dateTo = ''
    ): array {
        if ($this->useMongo) {
            return $this->getContactsWithProfilesMongo($search, $limit, $offset, $sortColumn, $sortOrder, $type, $dateFrom, $dateTo);
        }

        return $this->getContactsWithProfilesSQL($search, $limit, $offset, $sortColumn, $sortOrder);
    }

    private function getContactsWithProfilesSQL(
        string $search,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $whereClause = '1=1';
        $params = [];

        if ($search) {
            $whereClause = "(c.name LIKE ? OR c.email LIKE ? OR c.company LIKE ?)";
            $searchParam = "%{$search}%";
            $params = [$searchParam, $searchParam, $searchParam];
        }

        // Validate sort column
        $allowedColumns = ['name', 'email', 'company', 'created_at', 'reliability_score', 'risk_level'];
        if (!in_array($sortColumn, $allowedColumns)) {
            $sortColumn = 'name';
        }
        $sortOrder = strtoupper($sortOrder) === 'DESC' ? 'DESC' : 'ASC';

        $total = $this->db->fetchColumn(
            "SELECT COUNT(*) FROM contacts c WHERE {$whereClause}",
            $params
        );

        $contacts = $this->db->fetchAll(
            "SELECT c.*,
                cp.relationship_health, cp.reliability_score, cp.risk_level,
                cp.pricing_sensitivity, cp.ai_recommendations
             FROM contacts c
             LEFT JOIN contact_profiles cp ON c.id = cp.contact_id
             WHERE {$whereClause}
             ORDER BY {$sortColumn} {$sortOrder}
             LIMIT ? OFFSET ?",
            array_merge($params, [$limit, $offset])
        );

        return ['contacts' => $contacts, 'total' => (int)$total];
    }

    private function getContactsWithProfilesMongo(
        string $search,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder,
        string $type = '',
        string $dateFrom = '',
        string $dateTo = ''
    ): array {
        $filter = [];
        if ($search) {
            $filter['$or'] = [
                ['name' => ['$regex' => $search, '$options' => 'i']],
                ['email' => ['$regex' => $search, '$options' => 'i']],
                ['company' => ['$regex' => $search, '$options' => 'i']]
            ];
        }
        if ($type) {
            $filter['type'] = $type;
        }

        // Get total count
        $mongo = MongoDatabase::getInstance();

        // Map sort column to MongoDB field
        $sortDir = strtoupper($sortOrder) === 'DESC' ? -1 : 1;

        // Get contacts with profiles via aggregation
        $pipeline = [];

        if (!empty($filter)) {
            $pipeline[] = ['$match' => $filter];
        }

        $pipeline[] = ['$lookup' => [
            'from' => 'contact_profiles',
            'localField' => 'id',
            'foreignField' => 'contact_id',
            'as' => 'profile'
        ]];

        $pipeline[] = ['$unwind' => [
            'path' => '$profile',
            'preserveNullAndEmptyArrays' => true
        ]];

        // For last_contact sort or date filters, lookup latest email date
        if ($sortColumn === 'last_contact' || $dateFrom || $dateTo) {
            $pipeline[] = ['$lookup' => [
                'from' => 'emails',
                'let' => ['cid' => '$id'],
                'pipeline' => [
                    ['$match' => ['$expr' => ['$eq' => ['$contact_id', '$$cid']]]],
                    ['$group' => ['_id' => null, 'max_date' => ['$max' => '$date']]]
                ],
                'as' => 'email_stats'
            ]];
            $pipeline[] = ['$addFields' => [
                'last_contact' => ['$ifNull' => [
                    ['$arrayElemAt' => ['$email_stats.max_date', 0]],
                    '1970-01-01'
                ]]
            ]];
            $pipeline[] = ['$project' => ['email_stats' => 0]];

            // Apply date range filter on last_contact
            if ($dateFrom || $dateTo) {
                $dateFilter = [];
                if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
                if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
                $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
            }
        }

        // Count after all filters (need separate pipeline for accurate count with date filters)
        if ($dateFrom || $dateTo) {
            $countPipeline = $pipeline;
            $countPipeline[] = ['$count' => 'total'];
            $countResult = $mongo->aggregate('contacts', $countPipeline);
            $total = $countResult[0]['total'] ?? 0;
        } else {
            $total = $mongo->count('contacts', $filter);
        }

        $pipeline[] = ['$sort' => [$sortColumn => $sortDir]];
        $pipeline[] = ['$skip' => $offset];
        $pipeline[] = ['$limit' => $limit];

        $results = $mongo->aggregate('contacts', $pipeline);

        // Flatten profile fields
        $contacts = array_map(function($doc) {
            if (isset($doc['profile'])) {
                $doc['relationship_health'] = $doc['profile']['relationship_health'] ?? null;
                $doc['reliability_score'] = $doc['profile']['reliability_score'] ?? null;
                $doc['risk_level'] = $doc['profile']['risk_level'] ?? null;
                $doc['pricing_sensitivity'] = $doc['profile']['pricing_sensitivity'] ?? null;
                $doc['ai_recommendations'] = $doc['profile']['ai_recommendations'] ?? null;
                unset($doc['profile']);
            }
            return $doc;
        }, $results);

        return ['contacts' => $contacts, 'total' => $total];
    }

    /**
     * Get single contact with full details
     */
    public function getContactWithDetails(int $contactId): ?array
    {
        if ($this->useMongo) {
            return $this->getContactWithDetailsMongo($contactId);
        }

        return $this->getContactWithDetailsSQL($contactId);
    }

    private function getContactWithDetailsSQL(int $contactId): ?array
    {
        $contact = $this->db->fetchOne(
            "SELECT c.*, cp.*
             FROM contacts c
             LEFT JOIN contact_profiles cp ON c.id = cp.contact_id
             WHERE c.id = ?",
            [$contactId]
        );

        if (!$contact) return null;

        // Get recent emails
        $recentEmails = $this->db->fetchAll(
            "SELECT e.id, e.subject, e.date, e.direction, e.snippet,
                    ea.sentiment_label, ea.urgency_level, ea.summary
             FROM emails e
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             WHERE e.contact_id = ?
             ORDER BY e.date DESC
             LIMIT 20",
            [$contactId]
        );

        // Get quotes
        $quotes = $this->db->fetchAll(
            "SELECT * FROM quotes WHERE contact_id = ? ORDER BY quote_date DESC",
            [$contactId]
        );

        // Parse JSON fields
        $jsonFields = ['personality_traits', 'detected_inconsistencies', 'risk_factors',
                      'best_approach', 'topics_to_avoid', 'quote_history'];
        foreach ($jsonFields as $field) {
            if (isset($contact[$field]) && is_string($contact[$field])) {
                $contact[$field] = json_decode($contact[$field], true);
            }
        }

        return [
            'contact' => $contact,
            'recent_emails' => $recentEmails,
            'quotes' => $quotes
        ];
    }

    private function getContactWithDetailsMongo(int $contactId): ?array
    {
        $mongo = MongoDatabase::getInstance();

        // Get contact with profile
        $pipeline = [
            ['$match' => ['id' => $contactId]],
            ['$lookup' => [
                'from' => 'contact_profiles',
                'localField' => 'id',
                'foreignField' => 'contact_id',
                'as' => 'profile'
            ]],
            ['$unwind' => ['path' => '$profile', 'preserveNullAndEmptyArrays' => true]]
        ];

        $results = $mongo->aggregate('contacts', $pipeline);
        if (empty($results)) return null;

        $contact = $results[0];

        // Flatten profile
        if (isset($contact['profile'])) {
            foreach ($contact['profile'] as $key => $value) {
                if ($key !== '_id' && $key !== 'contact_id') {
                    $contact[$key] = $value;
                }
            }
            unset($contact['profile']);
        }

        // Get recent emails with analysis
        $emailPipeline = [
            ['$match' => ['contact_id' => $contactId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'id',
                'foreignField' => 'email_id',
                'as' => 'analysis'
            ]],
            ['$unwind' => ['path' => '$analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => ['date' => -1]],
            ['$limit' => 20],
            ['$project' => [
                'id' => 1,
                'subject' => 1,
                'date' => 1,
                'direction' => 1,
                'snippet' => 1,
                'sentiment_label' => '$analysis.sentiment_label',
                'urgency_level' => '$analysis.urgency_level',
                'summary' => '$analysis.summary'
            ]]
        ];

        $recentEmails = $mongo->aggregate('emails', $emailPipeline);

        // Get quotes
        $quotes = $mongo->findAll('quotes',
            ['contact_id' => $contactId],
            ['sort' => ['quote_date' => -1]]
        );

        return [
            'contact' => $contact,
            'recent_emails' => $recentEmails,
            'quotes' => $quotes
        ];
    }

    // =========================================================================
    // EMAILS
    // =========================================================================

    /**
     * Get emails with analysis for a contact
     */
    public function getEmailsWithAnalysis(
        int $contactId,
        int $limit = 50,
        int $offset = 0,
        string $sortColumn = 'date',
        string $sortOrder = 'DESC'
    ): array {
        if ($this->useMongo) {
            return $this->getEmailsWithAnalysisMongo($contactId, $limit, $offset, $sortColumn, $sortOrder);
        }

        return $this->getEmailsWithAnalysisSQL($contactId, $limit, $offset, $sortColumn, $sortOrder);
    }

    private function getEmailsWithAnalysisSQL(
        int $contactId,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $total = $this->db->fetchColumn(
            "SELECT COUNT(*) FROM emails WHERE contact_id = ?",
            [$contactId]
        );

        $sortOrder = strtoupper($sortOrder) === 'DESC' ? 'DESC' : 'ASC';

        $emails = $this->db->fetchAll(
            "SELECT e.*, ea.sentiment_label, ea.urgency_level, ea.summary, ea.key_points
             FROM emails e
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             WHERE e.contact_id = ?
             ORDER BY e.{$sortColumn} {$sortOrder}
             LIMIT ? OFFSET ?",
            [$contactId, $limit, $offset]
        );

        return ['emails' => $emails, 'total' => (int)$total];
    }

    private function getEmailsWithAnalysisMongo(
        int $contactId,
        int $limit,
        int $offset,
        string $sortColumn,
        string $sortOrder
    ): array {
        $mongo = MongoDatabase::getInstance();

        $total = $mongo->count('emails', ['contact_id' => $contactId]);

        $sortDir = strtoupper($sortOrder) === 'DESC' ? -1 : 1;

        $pipeline = [
            ['$match' => ['contact_id' => $contactId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'id',
                'foreignField' => 'email_id',
                'as' => 'analysis'
            ]],
            ['$unwind' => ['path' => '$analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => [$sortColumn => $sortDir]],
            ['$skip' => $offset],
            ['$limit' => $limit]
        ];

        $results = $mongo->aggregate('emails', $pipeline);

        // Flatten analysis
        $emails = array_map(function($doc) {
            if (isset($doc['analysis'])) {
                $doc['sentiment_label'] = $doc['analysis']['sentiment_label'] ?? null;
                $doc['urgency_level'] = $doc['analysis']['urgency_level'] ?? null;
                $doc['summary'] = $doc['analysis']['summary'] ?? null;
                $doc['key_points'] = $doc['analysis']['key_points'] ?? null;
                unset($doc['analysis']);
            }
            return $doc;
        }, $results);

        return ['emails' => $emails, 'total' => $total];
    }

    /**
     * Get single email with full details
     */
    public function getEmailWithDetails(int $emailId): ?array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getEmailWithDetails($emailId);
        }

        return $this->db->fetchOne(
            "SELECT e.*, c.name as contact_name, c.email as contact_email,
                    ea.*, ta.summary as thread_summary, ta.key_insights
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             LEFT JOIN email_analysis ea ON e.id = ea.email_id
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE e.id = ?",
            [$emailId]
        );
    }

    /**
     * Search emails
     */
    public function searchEmails(
        string $query,
        ?int $contactId = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->searchEmails($query, [
                'contact_id' => $contactId,
                'limit' => $limit
            ]);
        }

        $whereClause = "(e.subject LIKE ? OR e.body_text LIKE ?)";
        $params = ["%{$query}%", "%{$query}%"];

        if ($contactId) {
            $whereClause .= " AND e.contact_id = ?";
            $params[] = $contactId;
        }

        return $this->db->fetchAll(
            "SELECT e.*, c.name as contact_name, c.email as contact_email
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY e.date DESC
             LIMIT ?",
            array_merge($params, [$limit])
        );
    }

    // =========================================================================
    // THREADS
    // =========================================================================

    /**
     * Get thread analysis with contact info
     */
    public function getThreadsWithContact(
        ?int $contactId = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $filter = $contactId ? ['contact_id' => $contactId] : [];
            $repo = new MongoRepository();
            return $repo->getThreadsWithContact($filter, ['limit' => $limit]);
        }

        $whereClause = $contactId ? "ta.contact_id = ?" : "1=1";
        $params = $contactId ? [$contactId, $limit] : [$limit];

        return $this->db->fetchAll(
            "SELECT ta.*, c.name as contact_name, c.email as contact_email
             FROM thread_analysis ta
             LEFT JOIN contacts c ON ta.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY ta.analyzed_at DESC
             LIMIT ?",
            $params
        );
    }

    /**
     * Get list of threads with summary information (paginated)
     *
     * @param int|null $contactId Filter by contact ID
     * @param string $search Search term for subject
     * @param int $limit Results per page
     * @param int $offset Pagination offset
     * @return array ['threads' => [...], 'total' => int]
     */
    public function getThreads(
        ?int $contactId = null,
        string $search = '',
        int $limit = 20,
        int $offset = 0,
        string $sort = 'last_date',
        string $order = 'DESC',
        string $contactName = '',
        string $dateFrom = '',
        string $dateTo = '',
        string $status = ''
    ): array {
        if ($this->useMongo) {
            return $this->getThreadsMongo($contactId, $search, $limit, $offset, $sort, $order, $contactName, $dateFrom, $dateTo, $status);
        }

        return $this->getThreadsSQL($contactId, $search, $limit, $offset);
    }

    private function getThreadsSQL(?int $contactId, string $search, int $limit, int $offset): array
    {
        $userEmail = defined('GMAIL_USER_EMAIL') ? strtolower(GMAIL_USER_EMAIL) : '';
        $where = 'e.thread_id IS NOT NULL AND e.thread_id != ""';
        $params = [];

        if ($contactId) {
            $where .= ' AND e.contact_id = ?';
            $params[] = $contactId;
        }

        if ($search) {
            $where .= ' AND e.subject LIKE ?';
            $params[] = '%' . $search . '%';
        }

        // Get total thread count
        $total = $this->db->fetchColumn(
            "SELECT COUNT(DISTINCT e.thread_id)
             FROM emails e
             WHERE {$where}",
            $params
        );

        // Get threads with summary info
        // Use subquery for contact_id to find most frequent inbound sender (not MAX)
        $threads = $this->db->fetchAll(
            "SELECT
                e.thread_id,
                MIN(e.id) as first_email_id,
                MAX(e.id) as last_email_id,
                MIN(e.subject) as subject,
                MIN(e.date) as first_date,
                MAX(e.date) as last_date,
                COUNT(*) as email_count,
                SUM(CASE WHEN e.direction = 'inbound' THEN 1 ELSE 0 END) as inbound_count,
                SUM(CASE WHEN e.direction = 'outbound' THEN 1 ELSE 0 END) as outbound_count,
                (SELECT e2.contact_id FROM emails e2
                 WHERE e2.thread_id = e.thread_id
                 AND e2.direction = 'inbound'
                 AND LOWER(e2.from_email) != ?
                 GROUP BY e2.contact_id
                 ORDER BY COUNT(*) DESC LIMIT 1) as contact_id,
                (SELECT c2.name FROM emails e2
                 JOIN contacts c2 ON e2.contact_id = c2.id
                 WHERE e2.thread_id = e.thread_id
                 AND e2.direction = 'inbound'
                 AND LOWER(e2.from_email) != ?
                 GROUP BY e2.contact_id, c2.name
                 ORDER BY COUNT(*) DESC LIMIT 1) as contact_name,
                ta.is_analyzed,
                ta.needs_reanalysis
             FROM emails e
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE {$where}
             GROUP BY e.thread_id
             ORDER BY MAX(e.date) DESC
             LIMIT ? OFFSET ?",
            array_merge([$userEmail, $userEmail], $params, [$limit, $offset])
        );

        return [
            'threads' => $threads,
            'total' => (int)$total
        ];
    }

    private function getThreadsMongo(
        ?int $contactId,
        string $search,
        int $limit,
        int $offset,
        string $sort = 'last_date',
        string $order = 'DESC',
        string $contactName = '',
        string $dateFrom = '',
        string $dateTo = '',
        string $status = ''
    ): array {
        $mongo = MongoDatabase::getInstance();
        $userEmail = defined('GMAIL_USER_EMAIL') ? strtolower(GMAIL_USER_EMAIL) : '';

        // Build match filter
        $matchFilter = [
            'thread_id' => ['$ne' => null, '$ne' => '']
        ];

        if ($contactId) {
            $matchFilter['contact_id'] = $contactId;
        }

        if ($search) {
            $matchFilter['subject'] = ['$regex' => $search, '$options' => 'i'];
        }

        // Date range filter on email dates
        if ($dateFrom || $dateTo) {
            $dateFilter = [];
            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
            $matchFilter['date'] = $dateFilter;
        }

        // Contact name filter: pre-query contact IDs
        if ($contactName) {
            $matchingContacts = $mongo->findAll('contacts', [
                'name' => ['$regex' => $contactName, '$options' => 'i']
            ], ['projection' => ['id' => 1]]);
            $matchingIds = array_column($matchingContacts, 'id');
            if (!empty($matchingIds)) {
                $matchFilter['contact_id'] = ['$in' => $matchingIds];
            } else {
                return ['threads' => [], 'total' => 0];
            }
        }

        // Count distinct threads for total
        if ($status) {
            // Need full pipeline to filter by analysis status
            $countPipeline = [
                ['$match' => $matchFilter],
                ['$group' => [
                    '_id' => '$thread_id',
                    'email_count' => ['$sum' => 1]
                ]],
                ['$lookup' => [
                    'from' => 'thread_analysis',
                    'localField' => '_id',
                    'foreignField' => 'thread_id',
                    'as' => 'analysis'
                ]],
                ['$addFields' => [
                    'is_analyzed' => ['$arrayElemAt' => ['$analysis.is_analyzed', 0]]
                ]]
            ];
            if ($status === 'analyzed') {
                $countPipeline[] = ['$match' => ['is_analyzed' => true]];
            } elseif ($status === 'not_analyzed') {
                $countPipeline[] = ['$match' => ['$or' => [['is_analyzed' => false], ['is_analyzed' => null]]]];
            }
            $countPipeline[] = ['$count' => 'total'];
            $countResult = $mongo->aggregate('emails', $countPipeline);
            $total = !empty($countResult) ? $countResult[0]['total'] : 0;
        } else {
            $countPipeline = [
                ['$match' => $matchFilter],
                ['$group' => ['_id' => '$thread_id']],
                ['$count' => 'total']
            ];
            $countResult = $mongo->aggregate('emails', $countPipeline);
            $total = !empty($countResult) ? $countResult[0]['total'] : 0;
        }

        // Get threads with aggregated data
        // Collect all contacts per thread to find the most relevant one (not just $max)
        $pipeline = [
            ['$match' => $matchFilter],
            ['$group' => [
                '_id' => '$thread_id',
                'first_email_id' => ['$min' => '$id'],
                'last_email_id' => ['$max' => '$id'],
                'subject' => ['$first' => '$subject'],
                'first_date' => ['$min' => '$date'],
                'last_date' => ['$max' => '$date'],
                'email_count' => ['$sum' => 1],
                'inbound_count' => ['$sum' => ['$cond' => [['$eq' => ['$direction', 'inbound']], 1, 0]]],
                'outbound_count' => ['$sum' => ['$cond' => [['$eq' => ['$direction', 'outbound']], 1, 0]]],
                'all_contacts' => ['$push' => [
                    'cid' => '$contact_id',
                    'from' => '$from_email',
                    'to' => '$to_email',
                    'dir' => '$direction'
                ]]
            ]],
            ['$lookup' => [
                'from' => 'thread_analysis',
                'localField' => '_id',
                'foreignField' => 'thread_id',
                'as' => 'analysis'
            ]],
            ['$project' => [
                '_id' => 0,
                'thread_id' => '$_id',
                'first_email_id' => 1,
                'last_email_id' => 1,
                'subject' => 1,
                'first_date' => 1,
                'last_date' => 1,
                'email_count' => 1,
                'inbound_count' => 1,
                'outbound_count' => 1,
                'all_contacts' => 1,
                'is_analyzed' => ['$arrayElemAt' => ['$analysis.is_analyzed', 0]],
                'needs_reanalysis' => ['$arrayElemAt' => ['$analysis.needs_reanalysis', 0]]
            ]],
        ];

        // Status filter (after analysis lookup)
        if ($status === 'analyzed') {
            $pipeline[] = ['$match' => ['is_analyzed' => true]];
        } elseif ($status === 'not_analyzed') {
            $pipeline[] = ['$match' => ['$or' => [['is_analyzed' => false], ['is_analyzed' => null]]]];
        }

        // Dynamic sort
        $allowedSorts = ['last_date', 'first_date', 'email_count', 'subject'];
        $sortField = in_array($sort, $allowedSorts) ? $sort : 'last_date';
        $sortDir = strtoupper($order) === 'ASC' ? 1 : -1;

        $pipeline[] = ['$sort' => [$sortField => $sortDir]];
        $pipeline[] = ['$skip' => $offset];
        $pipeline[] = ['$limit' => $limit];

        $threads = $mongo->aggregate('emails', $pipeline);

        // Post-process: find the most frequent external contact per thread
        $contactIds = [];
        foreach ($threads as &$thread) {
            $contactCounts = [];

            foreach ($thread['all_contacts'] ?? [] as $c) {
                $cid = $c['cid'] ?? null;
                $fromEmail = strtolower($c['from'] ?? '');
                $toEmail = strtolower($c['to'] ?? '');
                $dir = $c['dir'] ?? '';

                if (!$cid) continue;

                // For inbound: from_email is external sender (weight 2x)
                if ($dir === 'inbound' && $fromEmail && $fromEmail !== $userEmail) {
                    $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 2;
                }
                // For outbound: to_email is external recipient (weight 1x)
                elseif ($dir === 'outbound' && $toEmail && $toEmail !== $userEmail) {
                    $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 1;
                }
            }

            // Pick the most frequent contact
            if (!empty($contactCounts)) {
                arsort($contactCounts);
                $thread['contact_id'] = (int)array_key_first($contactCounts);
            } else {
                // Fallback: first non-null contact_id
                $thread['contact_id'] = null;
                foreach ($thread['all_contacts'] ?? [] as $c) {
                    if (!empty($c['cid'])) {
                        $thread['contact_id'] = (int)$c['cid'];
                        break;
                    }
                }
            }

            if ($thread['contact_id']) {
                $contactIds[] = $thread['contact_id'];
            }

            unset($thread['all_contacts']);
        }
        unset($thread);

        // Batch lookup contact names
        $contactNames = [];
        if (!empty($contactIds)) {
            $contacts = $mongo->findAll('contacts', [
                'id' => ['$in' => array_values(array_unique($contactIds))]
            ]);
            foreach ($contacts as $c) {
                $contactNames[$c['id']] = $c['name'] ?? '';
            }
        }

        // Assign contact names
        foreach ($threads as &$thread) {
            $thread['contact_name'] = $contactNames[$thread['contact_id'] ?? 0] ?? null;
        }
        unset($thread);

        return [
            'threads' => $threads,
            'total' => $total
        ];
    }

    /**
     * Get full thread details with all emails and analysis
     *
     * @param string $threadId Gmail thread ID
     * @return array ['thread_id', 'emails', 'analysis', 'email_count']
     */
    public function getThreadDetails(string $threadId): array
    {
        if ($this->useMongo) {
            return $this->getThreadDetailsMongo($threadId);
        }

        return $this->getThreadDetailsSQL($threadId);
    }

    private function getThreadDetailsSQL(string $threadId): array
    {
        // Get all emails in thread with contact info
        $emails = $this->db->fetchAll(
            "SELECT e.*, c.name as contact_name, c.email as contact_email
             FROM emails e
             LEFT JOIN contacts c ON e.contact_id = c.id
             WHERE e.thread_id = ?
             ORDER BY e.date ASC",
            [$threadId]
        );

        // Get thread analysis with email analysis
        $analysis = $this->db->fetchOne(
            "SELECT ta.*, ea.summary, ea.sentiment_label, ea.deception_score,
                    ea.raw_analysis, ea.key_points
             FROM thread_analysis ta
             LEFT JOIN email_analysis ea ON ta.last_email_id = ea.email_id
             WHERE ta.thread_id = ?",
            [$threadId]
        );

        return [
            'thread_id' => $threadId,
            'emails' => $emails,
            'email_count' => count($emails),
            'analysis' => $analysis
        ];
    }

    private function getThreadDetailsMongo(string $threadId): array
    {
        $mongo = MongoDatabase::getInstance();

        // Get all emails in thread with contact info using aggregation
        $emailPipeline = [
            ['$match' => ['thread_id' => $threadId]],
            ['$lookup' => [
                'from' => 'contacts',
                'localField' => 'contact_id',
                'foreignField' => 'id',
                'as' => 'contact'
            ]],
            ['$unwind' => ['path' => '$contact', 'preserveNullAndEmptyArrays' => true]],
            ['$sort' => ['date' => 1]],  // ASC
            ['$project' => [
                '_id' => 0,
                'id' => 1,
                'gmail_id' => 1,
                'thread_id' => 1,
                'contact_id' => 1,
                'subject' => 1,
                'from_email' => 1,
                'to_email' => 1,
                'date' => 1,
                'direction' => 1,
                'snippet' => 1,
                'body_text' => 1,
                'body_html' => 1,
                'has_attachments' => 1,
                'label_ids' => 1,
                'contact_name' => '$contact.name',
                'contact_email' => '$contact.email'
            ]]
        ];

        $emails = $mongo->aggregate('emails', $emailPipeline);

        // Get thread analysis with email analysis
        $analysisPipeline = [
            ['$match' => ['thread_id' => $threadId]],
            ['$lookup' => [
                'from' => 'email_analysis',
                'localField' => 'last_email_id',
                'foreignField' => 'email_id',
                'as' => 'email_analysis'
            ]],
            ['$unwind' => ['path' => '$email_analysis', 'preserveNullAndEmptyArrays' => true]],
            ['$project' => [
                '_id' => 0,
                'thread_id' => 1,
                'contact_id' => 1,
                'subject' => 1,
                'email_count' => 1,
                'first_date' => 1,
                'last_date' => 1,
                'last_email_id' => 1,
                'trust_score' => 1,
                'deception_score' => 1,
                'risk_level' => 1,
                'summary' => ['$ifNull' => ['$email_analysis.summary', null]],
                'sentiment_label' => ['$ifNull' => ['$email_analysis.sentiment_label', null]],
                'raw_analysis' => 1,
                'key_insights' => 1,
                'key_points' => ['$ifNull' => ['$email_analysis.key_points', null]],
                'action_items' => 1,
                'warnings' => 1,
                'analyzed_at' => 1,
                'tokens_used' => 1
            ]],
            ['$limit' => 1]
        ];

        $analysisResults = $mongo->aggregate('thread_analysis', $analysisPipeline);
        $analysis = !empty($analysisResults) ? $analysisResults[0] : null;

        return [
            'thread_id' => $threadId,
            'emails' => $emails,
            'email_count' => count($emails),
            'analysis' => $analysis
        ];
    }

    /**
     * Get thread participants with aggregated statistics
     *
     * @param string $threadId Gmail thread ID
     * @param string $userEmail Current user's email for 'self' detection
     * @return array ['participants' => [...], 'subject' => string]
     */
    public function getThreadParticipants(string $threadId, string $userEmail): array
    {

        if ($this->useMongo) {
            return $this->getThreadParticipantsMongo($threadId, $userEmail);
        }

        return $this->getThreadParticipantsSQL($threadId, $userEmail);
    }

    private function getThreadParticipantsSQL(string $threadId, string $userEmail): array
    {
        $userEmail = strtolower($userEmail);

        // Fetch ALL emails in the thread to extract participants from FROM, TO, and CC fields
        $emails = $this->db->fetchAll(
            "SELECT from_email, to_email, cc_email, date, direction, subject
             FROM emails
             WHERE thread_id = ?
             ORDER BY date ASC",
            [$threadId]
        );

        // Collect all unique participants with their stats
        $participantStats = [];
        $threadSubject = '';

        foreach ($emails as $email) {
            // Get subject from first email
            if (empty($threadSubject) && !empty($email['subject'])) {
                $threadSubject = $email['subject'];
            }

            $date = $email['date'] ?? null;
            $direction = $email['direction'] ?? '';

            // Extract FROM participant
            if (!empty($email['from_email'])) {
                $fromEmail = strtolower(trim($email['from_email']));
                if (!isset($participantStats[$fromEmail])) {
                    $participantStats[$fromEmail] = [
                        'email' => $fromEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$fromEmail]['sent_count']++;
                $participantStats[$fromEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$fromEmail]['first_email'] || $date < $participantStats[$fromEmail]['first_email']) {
                        $participantStats[$fromEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$fromEmail]['last_email'] || $date > $participantStats[$fromEmail]['last_email']) {
                        $participantStats[$fromEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract TO participant(s)
            if (!empty($email['to_email'])) {
                $toEmail = strtolower(trim($email['to_email']));
                if (!isset($participantStats[$toEmail])) {
                    $participantStats[$toEmail] = [
                        'email' => $toEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$toEmail]['received_count']++;
                $participantStats[$toEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$toEmail]['first_email'] || $date < $participantStats[$toEmail]['first_email']) {
                        $participantStats[$toEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$toEmail]['last_email'] || $date > $participantStats[$toEmail]['last_email']) {
                        $participantStats[$toEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract CC participant(s)
            if (!empty($email['cc_email'])) {
                // CC might be comma-separated, split it
                $ccEmails = array_map('trim', explode(',', $email['cc_email']));
                foreach ($ccEmails as $ccEmail) {
                    $ccEmail = strtolower($ccEmail);
                    if (empty($ccEmail)) continue;

                    if (!isset($participantStats[$ccEmail])) {
                        $participantStats[$ccEmail] = [
                            'email' => $ccEmail,
                            'sent_count' => 0,
                            'received_count' => 0,
                            'total_count' => 0,
                            'first_email' => $date,
                            'last_email' => $date
                        ];
                    }
                    $participantStats[$ccEmail]['received_count']++;
                    $participantStats[$ccEmail]['total_count']++;
                    if ($date) {
                        if (!$participantStats[$ccEmail]['first_email'] || $date < $participantStats[$ccEmail]['first_email']) {
                            $participantStats[$ccEmail]['first_email'] = $date;
                        }
                        if (!$participantStats[$ccEmail]['last_email'] || $date > $participantStats[$ccEmail]['last_email']) {
                            $participantStats[$ccEmail]['last_email'] = $date;
                        }
                    }
                }
            }
        }

        // Convert to array and sort by total_count
        $participants = array_values($participantStats);
        usort($participants, function($a, $b) {
            return $b['total_count'] - $a['total_count'];
        });

        // Enrich with contact data and determine participant type
        foreach ($participants as &$p) {
            // Determine participant type
            $p['participant_type'] = (strtolower($p['email']) === $userEmail) ? 'self' : 'external';

            // Calculate inbound/outbound based on direction and participant type
            if ($p['participant_type'] === 'self') {
                $p['inbound_count'] = $p['received_count'];
                $p['outbound_count'] = $p['sent_count'];
            } else {
                $p['inbound_count'] = $p['sent_count'];  // External sent TO us = inbound
                $p['outbound_count'] = $p['received_count'];  // We sent TO external = outbound (for us)
            }

            $p['email_count'] = $p['total_count'];

            // Get name guess from email
            $emailParts = explode('@', $p['email']);
            $namePart = $emailParts[0] ?? '';
            $nameGuess = ucfirst(explode('.', $namePart)[0]);

            // Fetch contact data from contacts table
            $contact = $this->db->fetchOne(
                "SELECT id, name, email, company, type FROM contacts WHERE email = ?",
                [$p['email']]
            );

            // Fetch enrichment data from contact_enrichment table
            $enrichment = null;
            if ($contact && !empty($contact['id'])) {
                $enrichment = $this->db->fetchOne(
                    "SELECT job_title, phone, mobile_phone, linkedin_url
                     FROM contact_enrichment
                     WHERE contact_id = ?",
                    [$contact['id']]
                );
            }

            if ($p['participant_type'] === 'external') {
                if ($contact) {
                    $p['contact_id'] = $contact['id'];
                    $p['name'] = $contact['name'] ?? $nameGuess;
                    $p['company'] = $contact['company'] ?? '';
                    $p['contact_type'] = $contact['type'] ?? null;

                    // Add enriched data
                    if ($enrichment) {
                        $p['job_title'] = $enrichment['job_title'] ?? null;
                        $p['phone'] = $enrichment['phone'] ?? $enrichment['mobile_phone'] ?? null;
                        $p['linkedin_url'] = $enrichment['linkedin_url'] ?? null;
                    }
                } else {
                    $p['contact_id'] = null;
                    $p['name'] = $nameGuess;
                    $p['company'] = $emailParts[1] ?? '';
                    $p['contact_type'] = null;
                }
            } else {
                $p['name'] = 'You';
                $p['company'] = '';
                $p['contact_id'] = $contact ? $contact['id'] : null;
                $p['contact_type'] = null;
            }

            // Remove temporary fields
            unset($p['sent_count'], $p['received_count'], $p['total_count']);
        }

        return [
            'participants' => $participants,
            'subject' => $threadSubject
        ];
    }

    private function getThreadParticipantsMongo(string $threadId, string $userEmail): array
    {

        $mongo = MongoDatabase::getInstance();
        $userEmail = strtolower($userEmail);

        // Fetch ALL emails in the thread to extract participants from FROM, TO, and CC fields
        $emails = $mongo->findAll('emails', ['thread_id' => $threadId]);

        // Collect all unique participants with their stats
        $participantStats = [];
        $threadSubject = '';

        foreach ($emails as $email) {
            // Get subject from first email
            if (empty($threadSubject) && !empty($email['subject'])) {
                $threadSubject = $email['subject'];
            }

            $date = $email['date'] ?? null;
            $direction = $email['direction'] ?? '';

            // Extract FROM participant
            if (!empty($email['from_email'])) {
                $fromEmail = strtolower(trim($email['from_email']));
                if (!isset($participantStats[$fromEmail])) {
                    $participantStats[$fromEmail] = [
                        'email' => $fromEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$fromEmail]['sent_count']++;
                $participantStats[$fromEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$fromEmail]['first_email'] || $date < $participantStats[$fromEmail]['first_email']) {
                        $participantStats[$fromEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$fromEmail]['last_email'] || $date > $participantStats[$fromEmail]['last_email']) {
                        $participantStats[$fromEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract TO participant(s)
            if (!empty($email['to_email'])) {
                $toEmail = strtolower(trim($email['to_email']));
                if (!isset($participantStats[$toEmail])) {
                    $participantStats[$toEmail] = [
                        'email' => $toEmail,
                        'sent_count' => 0,
                        'received_count' => 0,
                        'total_count' => 0,
                        'first_email' => $date,
                        'last_email' => $date
                    ];
                }
                $participantStats[$toEmail]['received_count']++;
                $participantStats[$toEmail]['total_count']++;
                if ($date) {
                    if (!$participantStats[$toEmail]['first_email'] || $date < $participantStats[$toEmail]['first_email']) {
                        $participantStats[$toEmail]['first_email'] = $date;
                    }
                    if (!$participantStats[$toEmail]['last_email'] || $date > $participantStats[$toEmail]['last_email']) {
                        $participantStats[$toEmail]['last_email'] = $date;
                    }
                }
            }

            // Extract CC participant(s)
            if (!empty($email['cc_email'])) {
                // CC might be comma-separated, split it
                $ccEmails = array_map('trim', explode(',', $email['cc_email']));
                foreach ($ccEmails as $ccEmail) {
                    $ccEmail = strtolower($ccEmail);
                    if (empty($ccEmail)) continue;

                    if (!isset($participantStats[$ccEmail])) {
                        $participantStats[$ccEmail] = [
                            'email' => $ccEmail,
                            'sent_count' => 0,
                            'received_count' => 0,
                            'total_count' => 0,
                            'first_email' => $date,
                            'last_email' => $date
                        ];
                    }
                    $participantStats[$ccEmail]['received_count']++;
                    $participantStats[$ccEmail]['total_count']++;
                    if ($date) {
                        if (!$participantStats[$ccEmail]['first_email'] || $date < $participantStats[$ccEmail]['first_email']) {
                            $participantStats[$ccEmail]['first_email'] = $date;
                        }
                        if (!$participantStats[$ccEmail]['last_email'] || $date > $participantStats[$ccEmail]['last_email']) {
                            $participantStats[$ccEmail]['last_email'] = $date;
                        }
                    }
                }
            }
        }

        // Convert to array and sort by total_count
        $participants = array_values($participantStats);
        usort($participants, function($a, $b) {
            return $b['total_count'] - $a['total_count'];
        });

        // Enrich with contact data and determine participant type
        foreach ($participants as &$p) {
            // Determine participant type
            $p['participant_type'] = (strtolower($p['email']) === $userEmail) ? 'self' : 'external';

            // Calculate inbound/outbound based on direction and participant type
            if ($p['participant_type'] === 'self') {
                $p['inbound_count'] = $p['received_count'];
                $p['outbound_count'] = $p['sent_count'];
            } else {
                $p['inbound_count'] = $p['sent_count'];  // External sent TO us = inbound
                $p['outbound_count'] = $p['received_count'];  // We sent TO external = outbound (for us)
            }

            $p['email_count'] = $p['total_count'];

            // Get name guess from email
            $emailParts = explode('@', $p['email']);
            $namePart = $emailParts[0] ?? '';
            $nameGuess = ucfirst(explode('.', $namePart)[0]);

            // Fetch contact data from contacts collection
            $contact = $mongo->findOne('contacts', ['email' => $p['email']]);

            // Fetch enrichment data from contact_enrichment collection
            $enrichment = null;
            if ($contact && !empty($contact['id'])) {
                $enrichment = $mongo->findOne('contact_enrichment', ['contact_id' => $contact['id']]);
            }

            if ($p['participant_type'] === 'external') {
                if ($contact) {
                    $p['contact_id'] = $contact['id'] ?? null;
                    $p['name'] = $contact['name'] ?? $nameGuess;
                    $p['company'] = $contact['company'] ?? '';
                    $p['contact_type'] = $contact['type'] ?? null;

                    // Add enriched data
                    if ($enrichment) {
                        $p['job_title'] = $enrichment['job_title'] ?? null;
                        $p['phone'] = $enrichment['phone'] ?? $enrichment['mobile_phone'] ?? null;
                        $p['linkedin_url'] = $enrichment['linkedin_url'] ?? null;
                    }
                } else {
                    $p['contact_id'] = null;
                    $p['name'] = $nameGuess;
                    $p['company'] = $emailParts[1] ?? '';
                    $p['contact_type'] = null;
                }
            } else {
                $p['name'] = 'You';
                $p['company'] = '';
                $p['contact_id'] = $contact['id'] ?? null;
                $p['contact_type'] = null;
            }

            // Remove temporary fields
            unset($p['sent_count'], $p['received_count'], $p['total_count']);
        }

        return [
            'participants' => $participants,
            'subject' => $threadSubject
        ];
    }

    // =========================================================================
    // LOGS
    // =========================================================================

    /**
     * Get API logs with optional filtering
     *
     * @param string|null $threadId Filter by thread ID
     * @param string|null $level Filter by log level (info, warning, error)
     * @param int $limit Maximum number of logs to return
     * @return array ['logs' => [...], 'count' => int]
     */
    public function getLogs(?string $threadId = null, ?string $level = null, int $limit = 50): array
    {
        if ($this->useMongo) {
            return $this->getLogsMongo($threadId, $level, $limit);
        }

        return $this->getLogsSQL($threadId, $level, $limit);
    }

    private function getLogsSQL(?string $threadId, ?string $level, int $limit): array
    {
        $where = '1=1';
        $params = [];

        if ($threadId) {
            $where .= ' AND thread_id = ?';
            $params[] = $threadId;
        }
        if ($level) {
            $where .= ' AND log_level = ?';
            $params[] = $level;
        }

        $logs = $this->db->fetchAll(
            "SELECT id, log_level, source, message, context, thread_id, created_at
             FROM api_logs
             WHERE {$where}
             ORDER BY created_at DESC
             LIMIT ?",
            array_merge($params, [$limit])
        );

        // Parse context JSON (MySQL stores as string)
        foreach ($logs as &$log) {
            if (isset($log['context']) && is_string($log['context'])) {
                $log['context'] = json_decode($log['context'], true);
            }
        }

        return [
            'logs' => $logs,
            'count' => count($logs)
        ];
    }

    private function getLogsMongo(?string $threadId, ?string $level, int $limit): array
    {
        $mongo = MongoDatabase::getInstance();

        $filter = [];

        if ($threadId) {
            $filter['thread_id'] = $threadId;
        }
        if ($level) {
            $filter['log_level'] = $level;
        }

        $logs = $mongo->findAll('api_logs', $filter, [
            'sort' => ['created_at' => -1],
            'limit' => $limit
        ]);

        // Context should already be an array in MongoDB, but handle string case for safety
        foreach ($logs as &$log) {
            if (isset($log['context']) && is_string($log['context'])) {
                $log['context'] = json_decode($log['context'], true);
            }
        }

        return [
            'logs' => $logs,
            'count' => count($logs)
        ];
    }

    // =========================================================================
    // ALERTS
    // =========================================================================

    /**
     * Get alerts with contact info
     */
    public function getAlertsWithContact(
        ?string $status = null,
        int $limit = 50
    ): array {
        if ($this->useMongo) {
            $filter = $status ? ['status' => $status] : [];
            $repo = new MongoRepository();
            return $repo->getAlertsWithContact($filter, ['limit' => $limit]);
        }

        $whereClause = $status ? "a.status = ?" : "1=1";
        $params = $status ? [$status, $limit] : [$limit];

        return $this->db->fetchAll(
            "SELECT a.*, c.name as contact_name, c.email as contact_email
             FROM alerts a
             LEFT JOIN contacts c ON a.contact_id = c.id
             WHERE {$whereClause}
             ORDER BY a.created_at DESC
             LIMIT ?",
            $params
        );
    }

    // =========================================================================
    // BUSINESS INTELLIGENCE
    // =========================================================================

    /**
     * Get churn risk indicators with contacts
     */
    public function getChurnRisk(int $limit = 20): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getChurnRiskWithContacts(['limit' => $limit]);
        }

        return $this->db->fetchAll(
            "SELECT cri.*, c.name, c.email, c.company
             FROM churn_risk_indicators cri
             JOIN contacts c ON cri.contact_id = c.id
             ORDER BY cri.risk_score DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Get business opportunities with contacts
     */
    public function getBusinessOpportunities(int $limit = 20): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getBusinessOpportunities([], ['limit' => $limit]);
        }

        return $this->db->fetchAll(
            "SELECT bo.*, c.name, c.email, c.company
             FROM business_opportunities bo
             JOIN contacts c ON bo.contact_id = c.id
             ORDER BY bo.created_at DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Get leverage/bargaining chips
     */
    public function getLeverage(int $contactId, int $limit = 10): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getLeverageWithContacts(
                ['contact_id' => $contactId],
                ['limit' => $limit]
            );
        }

        return $this->db->fetchAll(
            "SELECT lb.*, c.name as contact_name
             FROM leverage_bargaining lb
             LEFT JOIN contacts c ON lb.contact_id = c.id
             WHERE lb.contact_id = ?
             ORDER BY lb.strength DESC
             LIMIT ?",
            [$contactId, $limit]
        );
    }

    // =========================================================================
    // STATISTICS
    // =========================================================================

    /**
     * Get contact email statistics
     */
    public function getContactStats(int $contactId): array
    {
        if ($this->useMongo) {
            $repo = new MongoRepository();
            return $repo->getContactEmailStats($contactId);
        }

        return $this->db->fetchOne(
            "SELECT
                COUNT(*) as email_count,
                SUM(CASE WHEN direction = 'sent' THEN 1 ELSE 0 END) as sent_count,
                SUM(CASE WHEN direction = 'received' THEN 1 ELSE 0 END) as received_count,
                MIN(date) as first_email,
                MAX(date) as last_email
             FROM emails
             WHERE contact_id = ?",
            [$contactId]
        ) ?? ['email_count' => 0, 'sent_count' => 0, 'received_count' => 0];
    }

    // =========================================================================
    // RAW ACCESS (for custom queries)
    // =========================================================================

    /**
     * Get the underlying Database instance (for SQL)
     */
    public function getDatabase(): Database
    {
        return $this->db;
    }

    /**
     * Get the MongoRepository instance (for custom MongoDB queries)
     */
    public function getMongoRepository(): ?MongoRepository
    {
        return $this->mongoRepo;
    }

    /**
     * Delete a contact and all associated data (emails, analysis, profiles, memories, AI responses)
     */
    public function deleteContact(int $contactId): array
    {
        $mongo = MongoDatabase::getInstance();
        $result = [];

        // Get thread_ids for this contact's emails before deleting them
        $emailDocs = $mongo->findAll('emails', ['contact_id' => $contactId], ['projection' => ['thread_id' => 1]]);
        $threadIds = [];
        foreach ($emailDocs as $doc) {
            if (!empty($doc['thread_id'])) {
                $threadIds[$doc['thread_id']] = true;
            }
        }
        $threadIds = array_keys($threadIds);

        // Delete emails
        $result['emails'] = $mongo->delete('emails', ['contact_id' => $contactId]);

        // For each thread, check if any emails remain; if not, delete thread_analysis
        $result['thread_analysis'] = 0;
        foreach ($threadIds as $tid) {
            $remaining = $mongo->count('emails', ['thread_id' => $tid]);
            if ($remaining === 0) {
                $result['thread_analysis'] += $mongo->delete('thread_analysis', ['thread_id' => $tid]);
            }
        }

        // Delete contact profile
        $result['contact_profiles'] = $mongo->delete('contact_profiles', ['contact_id' => $contactId]);

        // Delete AI contact memories
        $result['ai_contact_memory'] = $mongo->delete('ai_contact_memory', ['contact_id' => $contactId]);

        // Delete AI query responses for this contact
        $result['ai_query_responses'] = $mongo->delete('ai_query_responses', ['contact_id' => $contactId]);

        // Delete the contact record
        $result['contacts'] = $mongo->delete('contacts', ['id' => $contactId]);

        return $result;
    }

    /**
     * Delete a thread and all associated data (emails, analysis, AI responses)
     */
    public function deleteThread(string $threadId): array
    {
        $mongo = MongoDatabase::getInstance();
        $result = [];

        $result['emails'] = $mongo->delete('emails', ['thread_id' => $threadId]);
        $result['thread_analysis'] = $mongo->delete('thread_analysis', ['thread_id' => $threadId]);
        $result['ai_query_responses'] = $mongo->delete('ai_query_responses', ['thread_ids' => $threadId]);

        return $result;
    }

    /**
     * Delete all threads involving a specific email address or domain
     */
    public function deleteThreadsByEmail(string $email = '', string $domain = ''): array
    {
        $mongo = MongoDatabase::getInstance();

        // Build filter to find matching emails
        $filter = [];
        if ($email) {
            $emailLower = strtolower($email);
            $filter['$or'] = [
                ['from_email' => $emailLower],
                ['to_email' => $emailLower]
            ];
        } elseif ($domain) {
            $domainPattern = '@' . preg_quote($domain, '/') . '$';
            $filter['$or'] = [
                ['from_email' => ['$regex' => $domainPattern, '$options' => 'i']],
                ['to_email' => ['$regex' => $domainPattern, '$options' => 'i']]
            ];
        }

        // Aggregate unique thread_ids
        $pipeline = [
            ['$match' => $filter],
            ['$group' => ['_id' => '$thread_id']]
        ];
        $threadResults = $mongo->aggregate('emails', $pipeline);
        $threadIds = array_column($threadResults, '_id');

        $result = ['thread_count' => count($threadIds), 'emails' => 0, 'thread_analysis' => 0, 'ai_query_responses' => 0];

        foreach ($threadIds as $tid) {
            $r = $this->deleteThread($tid);
            $result['emails'] += $r['emails'] ?? 0;
            $result['thread_analysis'] += $r['thread_analysis'] ?? 0;
            $result['ai_query_responses'] += $r['ai_query_responses'] ?? 0;
        }

        return $result;
    }
}

-        // For last_contact sort or date filters, lookup latest email date

-        if ($sortColumn === 'last_contact' || $dateFrom || $dateTo) {

-            $pipeline[] = ['$lookup' => [

-            $pipeline[] = ['$addFields' => [

-                'last_contact' => ['$ifNull' => [

-                    ['$arrayElemAt' => ['$email_stats.max_date', 0]],

-            $pipeline[] = ['$project' => ['email_stats' => 0]];

-            // Apply date range filter on last_contact

-            if ($dateFrom || $dateTo) {

-                if ($dateFrom) $dateFilter['$gte'] = $dateFrom;

-                if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';

-                $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];

+        // Date range filter on last_contact (already stored on contact document)

+            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;

+            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';

+            $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];


Also need to fix the count — when date filters are applied, the count pipeline used the lookup too. Let me simplify that:


   198→            if (isset($doc['profile'])) {
   199→                $doc['relationship_health'] = $doc['profile']['relationship_health'] ?? null;
   200→                $doc['reliability_score'] = $doc['profile']['reliability_score'] ?? null;
   201→                $doc['risk_level'] = $doc['profile']['risk_level'] ?? null;
   202→                $doc['pricing_sensitivity'] = $doc['profile']['pricing_sensitivity'] ?? null;
   203→                $doc['ai_recommendations'] = $doc['profile']['ai_recommendations'] ?? null;
   204→                unset($doc['profile']);
   205→            }
   206→            return $doc;
   207→        }, $results);
   208→
   209→        return ['contacts' => $contacts, 'total' => $total];
   210→    }
   211→
   212→    /**
   213→     * Get single contact with full details
   214→     */
   215→    public function getContactWithDetails(int $contactId): ?array
   216→    {
   217→        if ($this->useMongo) {

            if (isset($doc['profile'])) {
                $doc['relationship_health'] = $doc['profile']['relationship_health'] ?? null;
                $doc['reliability_score'] = $doc['profile']['reliability_score'] ?? null;
                $doc['risk_level'] = $doc['profile']['risk_level'] ?? null;
                $doc['pricing_sensitivity'] = $doc['profile']['pricing_sensitivity'] ?? null;
                $doc['ai_recommendations'] = $doc['profile']['ai_recommendations'] ?? null;
                unset($doc['profile']);
            }
            return $doc;
        }, $results);

        return ['contacts' => $contacts, 'total' => $total];
    }

    /**
     * Get single contact with full details
     */
    public function getContactWithDetails(int $contactId): ?array
    {
        if ($this->useMongo) {


Now let me check the count logic:


   173→        if ($dateFrom || $dateTo) {
   174→            $dateFilter = [];
   175→            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
   176→            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
   177→            $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
   178→        }
   179→
   180→        // Count after all filters (need separate pipeline for accurate count with date filters)
   181→        if ($dateFrom || $dateTo) {
   182→            $countPipeline = $pipeline;
   183→            $countPipeline[] = ['$count' => 'total'];
   184→            $countResult = $mongo->aggregate('contacts', $countPipeline);
   185→            $total = $countResult[0]['total'] ?? 0;
   186→        } else {
   187→            $total = $mongo->count('contacts', $filter);
   188→        }
   189→
   190→        $pipeline[] = ['$sort' => [$sortColumn => $sortDir]];
   191→        $pipeline[] = ['$skip' => $offset];
   192→        $pipeline[] = ['$limit' => $limit];
   193→
   194→        $results = $mongo->aggregate('contacts', $pipeline);
   195→
   196→        // Flatten profile fields
   197→        $contacts = array_map(function($doc) {

        if ($dateFrom || $dateTo) {
            $dateFilter = [];
            if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
            if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
            $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
        }

        // Count after all filters (need separate pipeline for accurate count with date filters)
        if ($dateFrom || $dateTo) {
            $countPipeline = $pipeline;
            $countPipeline[] = ['$count' => 'total'];
            $countResult = $mongo->aggregate('contacts', $countPipeline);
            $total = $countResult[0]['total'] ?? 0;
        } else {
            $total = $mongo->count('contacts', $filter);
        }

        $pipeline[] = ['$sort' => [$sortColumn => $sortDir]];
        $pipeline[] = ['$skip' => $offset];
        $pipeline[] = ['$limit' => $limit];

        $results = $mongo->aggregate('contacts', $pipeline);

        // Flatten profile fields
        $contacts = array_map(function($doc) {


The count logic is fine — when no date filters, it uses simple count. When date filters exist, it copies the pipeline (which now only has `$match` on `last_contact`, no expensive lookup) and counts.


Bash

php -l /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php 2>&1

Check PHP syntax


No syntax errors detected in /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php


curl -s "https://mssql.danmarcrm.com/dev1/extension2v1/public/api.php?action=contacts&page=1&limit=5&sort=last_contact&order=DESC" 2>&1 | python3 -c "import sys,json; data=json.load(sys.stdin); print('Total:', data['pagination']['total'], 'Returned:', len(data['contacts'])); [print(c.get('name','?'), '-', c.get('email','?'), '-', c.get('last_contact','?')) for c in data['contacts']]"

Test contacts with last_contact sort after fix


Total: 288 Returned: 5
Andrew Dunbar - andrew@digitalj2.com - 2025-12-16 23:46:38
Iulian Oprescu - iulian.oprescu@f21digital.ro - 2025-12-16 23:14:08
Cullen McPhilliamy - cullen@digitalj2.com - 2025-12-16 22:53:50
Caroline Loor - caroline@flinsco.com - 2025-12-16 00:29:45
Ale Quarteiro - ale@flinsco.com - 2025-12-16 00:22:31


Conversation compacted


This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.

Analysis:
Let me chronologically analyze the conversation:

1. **Session Start**: This is a continuation from a previous conversation. The summary describes extensive prior work on an Email Intelligence System at `/var/www/mssql.danmarcrm.com/dev1/extension2v1/`. The previous session completed:
   - Two-Round AI feature implementation (mostly complete, needed wiring)
   - Thread view enhancements (agents, context builder, tokens, logging)
   - Dashboard fixes (hide analysis, reduce limits, fix contact name bug)
   - Bug fixes for PHP memory exhaustion, token estimation, content truncation

2. **First task - Wire Round 1 to Round 2**: The assistant completed the remaining Two-Round AI wiring in `submitToAI()` at thread_view.php lines 2931-2954, adding:
   - Store `lastRound1Providers`, `lastRound1Models`, `lastRound1Question`
   - Pass `'Round 1'` label to `displayAIResponse(data, 'Round 1')`
   - Check two-round checkbox and trigger `sendRound2()` automatically
   - `return` to skip the `finally` block (since `sendRound2` has its own `.finally()`)

3. **User's big request**: User provided 10 enhancement requests:
   - Delete functionality (contacts, threads, AI responses)
   - Sort by last activity
   - Filters for Contacts and Threads
   - Fix Etienne Turner thread linkage
   - Unwanted email threads investigation
   - Sync by domain/email
   - Delete all threads for email addresses
   - Two-Round AI chart functionality (D3.js)
   - Products tab (hide it)
   - Update AI model versions

4. **Planning phase**: The assistant entered plan mode, launched 3 explore agents in parallel and 1 plan agent. User answered clarification questions:
   - Products tab: "Hide it (keep code)"
   - Sync trigger: "Dashboard with Gmail API"
   - Chart type: "D3.js interactive"
   - Auto-sync: "Both options" (manual toggle + bulk delete)

5. **Implementation of 10 tasks across 6 phases**:

   **Phase 1 - Task 9 (Products tab)**: Commented out Products tab button in index.php

   **Phase 1 - Task 10 (AI models)**: Updated 4 files with Claude 4.6 models:
   - extension2v1 api_keys.php - added claude-opus-4-6, claude-sonnet-4-6
   - extension2 api_keys.php - same
   - thread_view.php MODEL_LIMITS - added new entries
   - config.php - CLAUDE_MODEL updated

   **Phase 2 - Task 1 (Delete)**:
   - QueryAdapter.php: Added `deleteContact()`, `deleteThread()`, `deleteThreadsByEmail()` methods
   - api.php: Added 4 endpoints (delete_contact, delete_thread, delete_ai_response, delete_threads_by_email)
   - app.js: Added delete buttons to contacts and threads tables, plus JS functions
   - thread_view.php: Added delete button to saved AI responses (both PHP-rendered and JS-loaded)

   **Phase 2 - Task 6 (Bulk delete)**: Included in Task 1 implementation above

   **Phase 3 - Task 2 (Sorting)**:
   - index.php: Added sort dropdowns for contacts and threads
   - app.js: Read sort values, pass to API
   - api.php: Pass sort/order params
   - QueryAdapter.php: Updated `getThreads()` and `getThreadsMongo()` to accept sort params

   **Phase 3 - Task 3 (Filters)**:
   - index.php: Added filter bars (date range, contact name, status)
   - app.js: Read filter values, pass to API, added clear functions
   - api.php: Pass filter params
   - QueryAdapter.php: Updated `getContactsWithProfilesMongo()` and `getThreadsMongo()` to accept filter params

   **Phase 4 - Task 4 (Etienne Turner fix)**:
   - Changed `all_contacts` $push to include `to_email` alongside `from_email`
   - Updated post-processing: for outbound emails, use `to_email` as external contact (weight 1x); for inbound, use `from_email` (weight 2x)

   **Phase 5 - Task 5 (Sync by domain)**:
   - api.php: Added `search_gmail_threads` and `import_gmail_threads` endpoints
   - index.php: Added "Import from Gmail" panel with search input
   - app.js: Added `searchByDomain()` and `importSelectedDomainThreads()` functions

   **Phase 5 - Task 8 (D3.js chart)**:
   - thread_view.php: Added D3.js v7 CDN, chart container, "Generate Chart" button
   - Added `generateChart()` and `renderD3Chart()` functions with force-directed graph, color-coded nodes, arrows, drag/zoom/pan, tooltips, legend

6. **Critical Bug**: User reported "contacts tab not showing any contact and not returning any after search"

7. **Bug Investigation and Fix**:
   - Tested API: `sort=last_contact&order=DESC` returned 288 total but empty contacts array
   - Tested API: `sort=name&order=ASC` worked fine → problem was in the `last_contact` sort pipeline
   - Root cause: The `$lookup` with sub-pipeline using `localField`/`foreignField` combined with `pipeline` was invalid MongoDB syntax
   - First fix attempt: Changed to `let`/`$expr` pattern → but this was too slow (timeout)
   - Found that contacts already have a `last_contact` field stored directly on the document
   - Final fix: Removed the expensive `$lookup` entirely, just sort by the existing `last_contact` field
   - Verified: API returns 5 contacts correctly sorted by last_contact DESC

Let me now check all the specific code changes and files modified.

Summary:
1. Primary Request and Intent:

This session is a continuation from a previous conversation. The session began by completing the Two-Round AI wiring from the prior session, then moved to a large batch of 10 enhancement requests from the user:

- **Delete functionality**: Allow deleting contacts, email threads, and AI responses
- **Sort by last activity**: Add sorting dropdowns to contacts and threads lists
- **Filters**: Add filter bars to Contacts and Threads tabs (date range, contact name, analysis status)
- **Fix Etienne Turner bug**: Not all threads should be linked to the same contact
- **Investigate unwanted threads**: Some threads synced automatically that user didn't request
- **Sync by domain/email**: Allow users to search Gmail by domain or email and import matching threads
- **Bulk delete by email**: Delete all threads for certain email addresses synced by mistake
- **D3.js interactive charts**: Add chart generation from AI responses with balloons, arrows, entities
- **Products tab**: Hide it (was implemented by mistake or wrong project)
- **Update AI model versions**: Claude 4.6 etc.

User clarification answers:
- Products tab → "Hide it (keep code)"
- Sync trigger → "Dashboard with Gmail API"
- Chart type → "D3.js interactive"
- Auto-sync → "Both options" (manual-only toggle + bulk delete)

2. Key Technical Concepts:
- MongoDB aggregation pipelines ($group, $lookup, $push, $match, $sort, $addFields)
- MongoDB `$lookup` with `let`/`$expr` vs `localField`/`foreignField` patterns
- MongoDatabase::delete() uses deleteMany internally, returns count
- Contact documents already store `last_contact` field — no need for expensive $lookup to emails collection
- D3.js v7 force-directed graphs with forceSimulation, forceLink, forceManyBody, forceCenter, forceCollide
- AbortController for JS fetch timeout handling
- GmailClient::searchThreads() accepts Gmail search query syntax (from:@domain OR to:@domain)
- QueryAdapter pattern: all data access goes through QueryAdapter, never raw SQL
- MongoDatabase::insertOrUpdate() for upserts (NOT upsert())
- AI proxy models keyed by provider: `[$provider => $model]`
- PHP `Database::getInstance()` used in api.php as `$db` — routes to MongoDB
- Contact `id` field is integer, used as foreign key in emails.contact_id

3. Files and Code Sections:

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/public/thread_view.php`**
  - Core thread view page with inline CSS, HTML, and JS
  - **Two-Round AI wiring** (lines ~2931-2954): Added code to store Round 1 state, display with label, check two-round checkbox, and auto-trigger `sendRound2()`:
    ```javascript
    // Store Round 1 info for potential Round 2 / follow-ups
    lastRound1Providers = selectedProvidersList.slice();
    lastRound1Models = Object.assign({}, modelsMap);
    lastRound1Question = actualQuestion;
    lastFullContent = fullContent;

    // Display Round 1 response
    displayAIResponse(data, 'Round 1');

    // Check if Two-Round mode is enabled → auto-trigger Round 2
    var twoRoundCheckbox = document.getElementById('two-round-mode');
    if (twoRoundCheckbox && twoRoundCheckbox.checked) {
        var r1SuccessCount = 0;
        for (var rk in lastRound1Responses) {
            if (lastRound1Responses[rk] && lastRound1Responses[rk].success) r1SuccessCount++;
        }
        if (r1SuccessCount > 0) {
            document.getElementById('loading-message').textContent = 'Round 1 complete (' + r1SuccessCount + ' responses). Starting Round 2 synthesis...';
            sendRound2(lastRound1Responses, lastRound1Providers, lastRound1Models, lastRound1Question);
            return; // Don't hide overlay — sendRound2's finally will do it
        }
    }
    ```
  - **Delete AI response button** added to saved responses (both PHP-rendered and dynamically loaded):
    ```html
    <button style="color:#ef4444;background:none;border:1px solid #ef4444;padding:4px 8px;border-radius:4px;cursor:pointer;font-size:0.85em;" onclick="event.stopPropagation(); deleteAiResponse('<?= $saved['_id'] ?>')">
        Del
    </button>
    ```
  - **`deleteAiResponse()` function** added:
    ```javascript
    async function deleteAiResponse(responseId) {
        if (!confirm('Delete this AI response? This cannot be undone.')) return;
        try {
            var response = await fetch('api.php?action=delete_ai_response', {
                method: 'POST',
                headers: {'Content-Type': 'application/json'},
                body: JSON.stringify({response_id: responseId})
            });
            var data = await response.json();
            if (data.success) {
                var el = document.getElementById('saved-response-' + responseId);
                if (el) el.remove();
            } else {
                alert('Error: ' + (data.error || 'Delete failed'));
            }
        } catch(e) {
            alert('Error deleting response: ' + e.message);
        }
    }
    ```
  - **MODEL_LIMITS updated** with new Claude 4.6 entries:
    ```javascript
    'claude-opus-4-6': 200000,
    'claude-sonnet-4-6': 200000,
    ```
  - **D3.js CDN** added: `<script src="https://d3js.org/d3.v7.min.js"></script>`
  - **"Generate Chart" button** added in follow-up bar:
    ```html
    <button class="btn btn-small" onclick="generateChart()" style="background:#8b5cf6;color:white;">Generate Chart</button>
    ```
  - **Chart container** added:
    ```html
    <div id="d3-chart-container" style="display:none; margin-bottom:20px;">
        <div class="card">
            <div class="card-header">
                <h2>Relationship Chart</h2>
                <button class="btn btn-secondary" onclick="document.getElementById('d3-chart-container').style.display='none'">Close</button>
            </div>
            <div id="d3-chart" style="width:100%;height:600px;background:#fafafa;..."></div>
            <div id="chart-tooltip" style="display:none;position:absolute;..."></div>
        </div>
    </div>
    ```
  - **`generateChart()` function**: Collects AI response text, sends to AI proxy asking for structured JSON `{nodes, links}`, parses response, calls `renderD3Chart()`
  - **`renderD3Chart(data)` function**: Full D3.js force-directed graph implementation with:
    - Color-coded nodes (person=#3b82f6, company=#10b981, topic=#f59e0b, event=#ef4444, action=#8b5cf6)
    - Arrow markers for directed links
    - Zoom/pan via `d3.zoom()`
    - Drag via `d3.drag()`
    - Hover tooltips
    - Link labels
    - Legend

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php`**
  - Core data access layer
  - **`getContactsWithProfiles()` signature updated** to accept `$type`, `$dateFrom`, `$dateTo`:
    ```php
    public function getContactsWithProfiles(
        string $search = '',
        int $limit = 50,
        int $offset = 0,
        string $sortColumn = 'name',
        string $sortOrder = 'ASC',
        string $type = '',
        string $dateFrom = '',
        string $dateTo = ''
    ): array
    ```
  - **`getContactsWithProfilesMongo()` updated**: Added type filter, date range filter on `last_contact` field (using the existing field on contact documents, NOT a $lookup):
    ```php
    if ($type) {
        $filter['type'] = $type;
    }
    // ...
    if ($dateFrom || $dateTo) {
        $dateFilter = [];
        if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
        if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
        $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
    }
    ```
  - **`getThreads()` signature updated** with 6 new params:
    ```php
    public function getThreads(
        ?int $contactId = null,
        string $search = '',
        int $limit = 20,
        int $offset = 0,
        string $sort = 'last_date',
        string $order = 'DESC',
        string $contactName = '',
        string $dateFrom = '',
        string $dateTo = '',
        string $status = ''
    ): array
    ```
  - **`getThreadsMongo()` updated** with sort/filter support:
    - Date range filter on `date` field in `$matchFilter`
    - Contact name pre-query: finds matching contact IDs, adds `$in` filter
    - Status filter after analysis lookup (`analyzed`/`not_analyzed`)
    - Dynamic sort on allowed fields (`last_date`, `first_date`, `email_count`, `subject`)
    - Separate count pipeline for status-filtered queries
  - **Etienne Turner fix**: Changed `all_contacts` $push to include `to_email`:
    ```php
    'all_contacts' => ['$push' => [
        'cid' => '$contact_id',
        'from' => '$from_email',
        'to' => '$to_email',
        'dir' => '$direction'
    ]]
    ```
    Post-processing updated:
    ```php
    foreach ($thread['all_contacts'] ?? [] as $c) {
        $cid = $c['cid'] ?? null;
        $fromEmail = strtolower($c['from'] ?? '');
        $toEmail = strtolower($c['to'] ?? '');
        $dir = $c['dir'] ?? '';
        if (!$cid) continue;
        if ($dir === 'inbound' && $fromEmail && $fromEmail !== $userEmail) {
            $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 2;
        }
        elseif ($dir === 'outbound' && $toEmail && $toEmail !== $userEmail) {
            $contactCounts[$cid] = ($contactCounts[$cid] ?? 0) + 1;
        }
    }
    ```
  - **Three new delete methods**:
    ```php
    public function deleteContact(int $contactId): array
    // Cascades: emails, thread_analysis (only if no remaining emails), contact_profiles, ai_contact_memory, ai_query_responses, contacts

    public function deleteThread(string $threadId): array
    // Deletes: emails, thread_analysis, ai_query_responses

    public function deleteThreadsByEmail(string $email = '', string $domain = ''): array
    // Aggregates thread_ids matching email/domain, calls deleteThread for each
    ```

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/public/api.php`**
  - Central API router, ~5700+ lines
  - **6 new endpoints added** before the `default` case:
    - `search_gmail_threads` — Builds Gmail query, calls `GmailClient::searchThreads()`, checks which already imported
    - `import_gmail_threads` — Loops through thread_ids, calls `GmailClient::getThread()`, saves emails/contacts using same pattern as `sync_thread`
    - `delete_contact` — Calls `QueryAdapter::deleteContact()`
    - `delete_thread` — Calls `QueryAdapter::deleteThread()`
    - `delete_ai_response` — Deletes from `ai_query_responses` by ObjectId
    - `delete_threads_by_email` — Calls `QueryAdapter::deleteThreadsByEmail()`
  - **Contacts endpoint updated**: Now reads `$type`, `$dateFrom`, `$dateTo` from GET params and passes to `getContactsWithProfiles()`
  - **Threads endpoint updated**: Now reads `$threadSort`, `$threadOrder`, `$threadContactName`, `$threadDateFrom`, `$threadDateTo`, `$threadStatus` and passes to `getThreads()`

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/public/assets/app.js`**
  - Dashboard JavaScript
  - **`loadContacts()` updated**: Reads sort dropdown value, date filters, passes to API
  - **`loadEmails()` updated**: Reads sort/filter values (thread-sort, thread-contact-filter, thread-date-from/to, thread-status-filter), passes to API
  - **Delete buttons added**: In contacts Actions column and new Actions column for threads
  - **New functions added**:
    - `deleteContact(contactId, contactName)` — with confirmation dialog
    - `deleteThread(threadId, subject)` — with confirmation dialog
    - `deleteThreadsByEmail(email, domain)` — bulk delete with confirmation
    - `promptBulkDeleteByEmail()` — prompts for email/domain input
    - `clearContactFilters()` — resets all contact filters
    - `clearThreadFilters()` — resets all thread filters
    - `searchByDomain()` — searches Gmail, displays results with checkboxes
    - `importSelectedDomainThreads()` — imports checked threads
  - **Sort change listeners** added via `DOMContentLoaded`

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/public/index.php`**
  - Dashboard HTML
  - **Products tab hidden**: `<!-- <button class="tab-btn" data-tab="products">🛒 Products</button> -->`
  - **Contacts tab enhanced**: Added sort dropdown (`contact-sort`), filter bar with date range + Apply/Clear buttons, "Bulk Delete by Email/Domain" button, "Import from Gmail" panel with search input
  - **Threads tab enhanced**: Added sort dropdown (`thread-sort`), filter bar with contact name, date range, status dropdown + Apply/Clear buttons

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/config/config.php`**
  - Updated: `define('CLAUDE_MODEL', 'claude-sonnet-4-6');`

- **`/var/www/mssql.danmarcrm.com/dev1/extension2v1/aiapi/api_opus/config/api_keys.php`**
  - Added `claude-opus-4-6` and `claude-sonnet-4-6` to models, pricing, set default to `claude-sonnet-4-6`

- **`/var/www/mssql.danmarcrm.com/dev1/extension2/aiapi/api_opus/config/api_keys.php`**
  - Same Claude 4.6 model updates applied to keep in sync

4. Errors and Fixes:

- **Contacts tab returning empty results (CRITICAL BUG)**:
  - User reported: "contacts tab not showing any contact and not returning any after search"
  - Root cause investigation: API returned `{"contacts": [], "pagination": {"total": 288, ...}}` when using `sort=last_contact`
  - Confirmed `sort=name` worked fine → issue was in the `last_contact` sort pipeline
  - **First fix attempt**: Used `$lookup` with `localField`/`foreignField` combined with `pipeline` sub-array — this was invalid MongoDB syntax (you can't mix both). Changed to `let`/`$expr` pattern.
  - **Second issue**: The `let`/`$expr` lookup worked but was extremely slow (timed out) — doing a full collection scan for each of 288 contacts across potentially thousands of emails.
  - **Discovery**: Contacts already have a `last_contact` field stored directly on the document (found via API test output showing `"last_contact": "2024-06-07 23:38:57"`)
  - **Final fix**: Removed the expensive `$lookup` entirely. Just sort by the existing `last_contact` field on contacts, and filter date ranges using `$match` on that same field.
  - Verified: API now returns 5 contacts correctly sorted by last_contact DESC

5. Problem Solving:

- **Two-Round AI completion**: Successfully wired `submitToAI()` → `displayAIResponse(data, 'Round 1')` → auto-trigger `sendRound2()` when checkbox is checked, with proper loading overlay management using `return` to skip the `finally` block
- **Contact name resolution**: Enhanced the MongoDB aggregation to include `to_email` for outbound emails, ensuring outbound-only threads can still resolve to the correct external contact
- **Contacts empty results**: Went through 3 iterations (invalid pipeline → slow lookup → direct field sort) to fix the `last_contact` sort, ultimately discovering the field already existed on contact documents

6. All User Messages:
- "working pretty well thank you - pls add functionality to delete: -- contacts -- email threads -- AI responses - allow sorting by last activity - add filters both in Contacts and Threads and add Last Activity date filter - need to be fixed as not all email threads to be linked to Etienne Turner - there are some email threads that for sure I didn't ask to be saved, where are those coming from? ex: https://mssql.danmarcrm.com/dev1/extension2v1/public/thread_view.php?thread_id=19bd4f343ca15018 - Very important, I need a functionality to allow users to sync by domain ex delta-engineering.be or by email address ddb@delta-engineering.be - we also need to be able to delete all email threads for certain email addresses if were sync by mistake - for Two-Round AI feature can we add a create chart functionality that will add events/things/persons/texts etc in balloons add arrows etc? - was the Products tab implemented by mistake? It may be asking for this in the wrong project, not sure if it makes sense here and if click Open Product Dashboard I get Access Denied from: https://mssql.danmarcrm.com/dev1/extension2v1/public/product_search.php - please update the AI models versions, Claude got to 4.6 already etc .. I think the proxy is also on this server if you need access to it? or to 46.xxx you can connect by ssh root@."
- User answered clarification questions: Products="Hide it (keep code)", Sync="Dashboard with Gmail API", Chart="D3.js interactive", Auto-sync="Both options"
- "contacts tab not showing any contact and not returning any after search"

7. Pending Tasks:
- **Task 7 (Chrome Extension manual-only mode)**: Documented but not implemented — extension files are on user's local Mac at `/Users/dan/Documents/extension2/extension2v1/chrome-extension`. Code specifications were provided but files cannot be edited from server.
- The contacts bug was just fixed — user hasn't confirmed it works in the UI yet.

8. Current Work:
The most recent work was fixing a critical bug where the contacts tab showed no contacts when using the default `last_contact` sort. The fix went through 3 iterations:
1. Invalid `$lookup` syntax → 2. Too-slow `$expr` lookup → 3. Just use the existing `last_contact` field on contact documents.

The final fix in `QueryAdapter.php` `getContactsWithProfilesMongo()` replaced an expensive `$lookup` with a simple date filter on the existing `last_contact` field:
```php
// Date range filter on last_contact (already stored on contact document)
if ($dateFrom || $dateTo) {
    $dateFilter = [];
    if ($dateFrom) $dateFilter['$gte'] = $dateFrom;
    if ($dateTo) $dateFilter['$lte'] = $dateTo . ' 23:59:59';
    $pipeline[] = ['$match' => ['last_contact' => $dateFilter]];
}
```

The fix was verified via API test returning 5 contacts correctly sorted. The user has not yet confirmed the fix works in their browser.

9. Optional Next Step:
Wait for user to confirm the contacts tab fix works in the UI. If there are additional issues with contacts display, search functionality, or any of the other 10 implemented features, address those next. The user's most recent message was "contacts tab not showing any contact and not returning any after search" which was just fixed and verified via API test.

If you need specific details from before compaction (like exact code snippets, error messages, or content you generated), read the full transcript at: /root/.claude/projects/-var-www-mssql-danmarcrm-com-dev1-extension2/75ef92ac-556a-4223-9d55-17c4e535294d.jsonl
Please continue the conversation from where we left off without asking the user any further questions. Continue with the last task that you were asked to work on.


Based on the summary, I just fixed the contacts tab bug where no contacts were showing when using the `last_contact` sort. The fix was verified via API test. Let me do a quick PHP syntax check on the modified file to make sure everything is solid, then summarize the current state for the user.


php -l /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/QueryAdapter.php 2>&1

PHP syntax check on QueryAdapter


php -l /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/api.php 2>&1

PHP syntax check on api.php


No syntax errors detected in /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/api.php


You're out of extra usage · resets 12am (UTC)


if I try to import delta-engineering.be it showed 30 existing and one new. I imported the missing one. If I search in threads by delta-engineering.be one ir retuned. If I search by Danny a lot are returned but paging doesn't seem to work .. click on 2-3 and nothing happens


The user is reporting issues with thread pagination/paging when filtering by contact name. Let me look at the relevant code to understand how pagination works with filters.


Let me investigate the pagination issue with thread filters.


81:            loadEmails();
388:    loadEmails(1, contactId);
654:async function loadEmails(page = 1, contactId = null) {
732:        renderPagination('emails-pagination', data.pagination, (p) => loadEmails(p));
4005:            loadEmails(state.emails.page);
4026:            loadEmails(state.emails.page);
4052:            loadEmails(state.emails.page);
4091:    loadEmails(1);
4099:    if (threadSort) threadSort.addEventListener('change', function() { loadEmails(1); });
4167:            loadEmails(1);


740:    // Redirect to dedicated thread view page with AI capabilities