166a167
>             <button class="tab-btn" data-tab="products">🛒 Products</button>
174a176
>             <button class="tab-btn" data-tab="business-ideas">💡 Business Ideas</button>
206a209,283
>
>

>

>
🛒 Product Search with AI

>
Search products from Amazon & AliExpress with advanced filters and AI recommendations

>

>
>
>

>

>

>                         Search Query
>                         <input type="text" id="quick-product-search" placeholder="e.g., LED strip lights COB waterproof" class="search-input" style="width: 100%;">
>

>

>                         IP Rating
>                         <select id="quick-filter-ip" class="search-input" style="width: 100%;">
>                             <option value="">Any</option>
>                             <option value="ip65">IP65</option>
>                             <option value="ip67">IP67</option>
>                             <option value="ip68">IP68</option>
>                             <option value="ip6+">IP6+</option>
>                         </select>
>

>

>                         Length
>                         <select id="quick-filter-length" class="search-input" style="width: 100%;">
>                             <option value="">Any</option>
>                             <option value="5m">5m</option>
>                             <option value="10m">10m</option>
>                             <option value="5m+">5m+</option>
>                             <option value="10m+">10m+</option>
>                         </select>
>

>

>                         <button class="btn btn-primary" onclick="window.location.href='product_search.php'" style="background: linear-gradient(135deg, #f59e0b 0%, #d97706 100%); border: none;">
>                             Open Full Search
>                         </button>
>

>

>

>
>
>

>

>
🎯 Smart Filters

>

>                         Filter by IP rating (IP65, IP 65, ip6, etc.), length (5m, 5 m, 10 meters), and more.
>                         Handles all format variations automatically.
>

>

>
>

>
🤖 AI Recommendations

>

>                         Send your product list to AI and get best picks categorized as Best Value, Top Rated, and Recommended.
>

>

>
>

>
➕ Custom Filters

>

>                         Add custom filters for COB type, wattage, voltage, or any text search. Extensible filter system.
>

>

>

>
>
>

>
>                     🔍 Open Product Search Dashboard
>
>

>

>
504,508c581,619
<
<

<

<                 ×
<

---
>
>

>

>
💡 Business Ideas Research

>
AI-powered research into business verticals, horizontals, and growth scenarios using web scraping and analysis

>

>
>
>

>
📊 Your Business Context

>
>                     Click to set/update your financial goals (helps AI provide better recommendations)
>

>

>                             Revenue Target
>                             <input type="text" id="owner-revenue-target" placeholder="e.g., $500K/year, 2x current revenue" style="width: 100%; padding: 10px; border: 1px solid #d1d5db; border-radius: 6px; box-sizing: border-box;">
>

>

>                             Investment Capacity
>                             <input type="text" id="owner-investment-capacity" placeholder="e.g., $50K available for new ventures" style="width: 100%; padding: 10px; border: 1px solid #d1d5db; border-radius: 6px; box-sizing: border-box;">
>

>

>                             Growth Objectives
>                             <input type="text" id="owner-growth-objectives" placeholder="e.g., Expand to new markets, diversify products" style="width: 100%; padding: 10px; border: 1px solid #d1d5db; border-radius: 6px; box-sizing: border-box;">
>

>

>                             Timeline
>                             <input type="text" id="owner-timeline" placeholder="e.g., 12-18 months" style="width: 100%; padding: 10px; border: 1px solid #d1d5db; border-radius: 6px; box-sizing: border-box;">
>

>

>                             Additional Notes
>                             <textarea id="owner-notes" placeholder="Any other relevant information about your business situation..." style="width: 100%; padding: 10px; border: 1px solid #d1d5db; border-radius: 6px; min-height: 60px; box-sizing: border-box;"></textarea>
>

>

>                             <button class="btn btn-primary" onclick="saveOwnerStatus()" style="background: #8b5cf6;">Save Business Context</button>
>
>

>

>
510d620
<

512,516c622,667
<
<

<

<                 ×
<

---
>
>

>
Start New Research Project

>

>                     Research Query
>                     <textarea id="business-ideas-query"
>                         placeholder="e.g., Research 50 business verticals related to LED lighting and smart home technology. For each vertical, identify 20-30 horizontal market segments. Focus on B2B opportunities, emerging markets, and areas with high growth potential."
>                         style="width: 100%; min-height: 100px; padding: 12px; border: 1px solid #d1d5db; border-radius: 6px; resize: vertical; box-sizing: border-box;"></textarea>
>

>
>
>
>                     Advanced Options
>

>

>

>                                 Target Verticals
>                                 <input type="number" id="param-verticals" value="50" min="1" max="100" style="width: 100%; padding: 8px; border: 1px solid #d1d5db; border-radius: 4px; box-sizing: border-box;">
>

>

>                                 Horizontals per Vertical
>                                 <input type="number" id="param-horizontals" value="25" min="1" max="50" style="width: 100%; padding: 8px; border: 1px solid #d1d5db; border-radius: 4px; box-sizing: border-box;">
>

>

>                                 Research Depth
>                                 <select id="param-depth" style="width: 100%; padding: 8px; border: 1px solid #d1d5db; border-radius: 4px; box-sizing: border-box;">
>                                     <option value="shallow">Shallow (Fast)</option>
>                                     <option value="medium" selected>Medium</option>
>                                     <option value="deep">Deep (Thorough)</option>
>                                 </select>
>

>

>                                 Scenario Depth
>                                 <select id="param-scenario-depth" style="width: 100%; padding: 8px; border: 1px solid #d1d5db; border-radius: 4px; box-sizing: border-box;">
>                                     <option value="high_level" selected>High-Level Strategic</option>
>                                     <option value="detailed">Detailed Action Plans</option>
>                                     <option value="financial">Financial Projections</option>
>                                 </select>
>

>

>

>
>
>                 <button class="btn btn-primary" onclick="createBusinessIdeasProject()" style="background: linear-gradient(135deg, #8b5cf6 0%, #6366f1 100%); padding: 12px 24px;">
>                     🚀 Start Research Project
>                 </button>
518d668
<

520,529c670,674
<
<

<
<

<

<                     🤖
<

<
AI Assistant

<
General Chat

<

---
>
>

>

>
0

>
Pending


The diffs are extensive. Let me understand the situation:

1. `extension2` has our recent changes (AI chat with agents, multi-thread, sessions, memory)
2. `extension2v1` is a different/newer version that has additional features (Products tab, Business Ideas tab, thread_view.php redirect, ImportantContext class, etc.) but does NOT have our AI chat enhancements

So I need to carefully apply only our AI chat changes to `extension2v1` without overwriting its unique features. Let me get the full picture of the differences first.

Let me look at more of the diffs to understand the full scope.


The diffs are significant. `extension2v1` is a **different/newer version** with extra features (Products tab, Business Ideas, thread_view.php, ImportantContext class) that `extension2` doesn't have. I need to carefully merge our AI chat changes INTO `extension2v1` without overwriting its unique features.

Let me get a complete picture of what needs to be ported.


TodoWrite

Analyze full diff scope between extension2 and extension2v1

Port ThreadAnalyzer.php changes (stripQuotedContent public static)

Port api.php changes (contacts fix, ai_chat rewrite, new endpoints)

Port app.js changes (context builder, agents, sessions, memory UI)

Port index.php changes (AI chat panel with agents, context builder)


Todos have been modified successfully. Ensure that you continue to use the todo list to track your progress. Please proceed with the current tasks if applicable

Analyze full diff scope between extension2 and extension2v1

Port ThreadAnalyzer.php changes (stripQuotedContent public static)

Port api.php changes (contacts fix, ai_chat rewrite, new endpoints)

Port app.js changes (context builder, agents, sessions, memory UI)

Port index.php changes (AI chat panel with agents, context builder)


Bash

diff /var/www/mssql.danmarcrm.com/dev1/extension2/public/api.php /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/api.php | wc -l

Count total diff lines for api.php


diff /var/www/mssql.danmarcrm.com/dev1/extension2/public/assets/app.js /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/assets/app.js | wc -l

Count total diff lines for app.js


diff /var/www/mssql.danmarcrm.com/dev1/extension2/public/index.php /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/index.php | wc -l

Count total diff lines for index.php


Line counts for all key files


  4416 /var/www/mssql.danmarcrm.com/dev1/extension2/public/api.php
  5094 /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/api.php
  3429 /var/www/mssql.danmarcrm.com/dev1/extension2/public/assets/app.js
  3195 /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/assets/app.js
   642 /var/www/mssql.danmarcrm.com/dev1/extension2/public/index.php
   766 /var/www/mssql.danmarcrm.com/dev1/extension2v1/public/index.php
 17542 total


These are large files with substantial diffs. The extension2v1 versions are actually bigger for api.php and index.php (more features), but extension2 has a bigger app.js (our AI chat additions).

I need to understand what specifically needs to be ported:

1. **ThreadAnalyzer.php**: `stripQuotedContent()` → public static + self:: call (2 line changes)
2. **api.php**:
   - `contacts` case: Replace raw SQL with QueryAdapter (extension2v1 still has the old SQL code)
   - `ai_chat` case: The entire rewrite with multi-thread, agents, sessions, memory
   - 6 new API endpoints for session and memory operations
   - Fix model proxy format to use `[$chatProvider => $chatModel]`
3. **app.js**:
   - AI provider constants, context builder functions, session and memory management
   - Modified sendAiChat() and initAiProviderDropdown()
4. **index.php**:
   - New AI chat panel HTML with agents, context builder, and session overlay
   - Provider and model dropdowns

Since the files are large and the v1 version is incomplete, I'm looking at what's actually missing from the target.

Examining ThreadAnalyzer.php first since it's the simplest change...


Large files with significant diffs. `extension2v1` has **extra features** (Products, Business Ideas, ImportantContext, thread_view.php redirect) that we must preserve. Let me read the specific sections I need to modify in each file.


1605:    private function stripQuotedContent(string $body): string


404:            $body = $this->stripQuotedContent($body);
1605:    private function stripQuotedContent(string $body): string


TodoWrite

Analyze full diff scope between extension2 and extension2v1

Port ThreadAnalyzer.php changes (stripQuotedContent public static)

Port api.php changes (contacts fix, ai_chat rewrite, new endpoints)

Port app.js changes (context builder, agents, sessions, memory UI)

Port index.php changes (AI chat panel with agents, context builder)


**1. ThreadAnalyzer.php** — Two simple changes:


The file /var/www/mssql.danmarcrm.com/dev1/extension2v1/src/ThreadAnalyzer.php has been updated successfully.

<?php
/**
 * Thread Analyzer - Analyzes Complete Email Threads
 *
 * Unlike AIAnalyzer which processes individual emails,
 * this analyzes ENTIRE email threads (conversations) together.
 *
 * WHY THREADS?
 * - Detects inconsistencies across multiple emails
 * - Understands negotiation flow (quote → counter → agreement)
 * - Sees how tone/sentiment changes through conversation
 * - Better psychological profiling with full context
 *
 * CALLING MAP:
 * - Called by: cli/analyze_threads.php (main entry)
 * - Called by: cli/cron_hybrid.php (scheduled)
 * - Uses: Database.php, calls Claude API
 * - Reads from: emails table (grouped by thread_id)
 * - Writes to: thread_analysis, email_analysis tables
 *
 * @package EmailIntelligence
 */

namespace EmailIntelligence;

use Exception;
use GuzzleHttp\Client as HttpClient;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Handler\CurlHandler;

class ThreadAnalyzer
{
    private Database $db;
    private HttpClient $http;
    private array $config;
    private $progressCallback = null;

    private array $stats = [
        'threads_analyzed' => 0,
        'threads_failed' => 0,
        'emails_processed' => 0,
        'quotes_extracted' => 0,
        'inconsistencies_found' => 0,
        'tokens_used' => 0,
        'start_time' => null
    ];

    public function __construct(?string $provider = null, ?string $model = null)
    {
        $this->db = Database::getInstance();

        // Determine which AI provider to use
        // Use passed parameter, fall back to constant, then default to 'claude'
        if ($provider === null) {
            $provider = defined('AI_PROVIDER') ? AI_PROVIDER : 'claude';
        }

        if ($provider === 'openai' && defined('OPENAI_API_KEY') && !empty(OPENAI_API_KEY)) {
            // Determine OpenAI model
            // If model is passed, use it; otherwise use OPENAI_MODEL constant or default
            $selectedModel = $model ?? (defined('OPENAI_MODEL') ? OPENAI_MODEL : 'gpt-4o');

            $this->config = [
                'provider' => 'openai',
                'api_key' => OPENAI_API_KEY,
                'model' => $selectedModel,
                'max_tokens' => defined('OPENAI_MAX_TOKENS') ? OPENAI_MAX_TOKENS : 16000,
                'endpoint' => 'https://api.openai.com/v1/chat/completions'
            ];

            // Create clean handler stack to avoid New Relic Guzzle infinite loop
            $stack = HandlerStack::create(new CurlHandler());

            $this->http = new HttpClient([
                'timeout' => 180,
                'handler' => $stack,
                'headers' => [
                    'Content-Type' => 'application/json',
                    'Authorization' => 'Bearer ' . $this->config['api_key']
                ]
            ]);
        } else {
            // Default to Claude
            // Determine Claude model
            // If model is passed, use it; otherwise use CLAUDE_MODEL constant or default
            $selectedModel = $model ?? (defined('CLAUDE_MODEL') ? CLAUDE_MODEL : 'claude-sonnet-4-5-20250929');

            $this->config = [
                'provider' => 'claude',
                'api_key' => defined('CLAUDE_API_KEY') ? CLAUDE_API_KEY : '',
                'model' => $selectedModel,
                'max_tokens' => defined('CLAUDE_MAX_TOKENS') ? CLAUDE_MAX_TOKENS : 16000,
                'endpoint' => 'https://api.anthropic.com/v1/messages'
            ];

            // Create clean handler stack to avoid New Relic Guzzle infinite loop
            $stack = HandlerStack::create(new CurlHandler());

            $this->http = new HttpClient([
                'timeout' => 180,
                'handler' => $stack,
                'headers' => [
                    'Content-Type' => 'application/json',
                    'x-api-key' => $this->config['api_key'],
                    'anthropic-version' => '2023-06-01'
                ]
            ]);
        }
    }

    public function isConfigured(): bool
    {
        return !empty($this->config['api_key']);
    }

    public function getProvider(): string
    {
        return $this->config['provider'] ?? 'claude';
    }

    public function setProgressCallback(callable $callback): void
    {
        $this->progressCallback = $callback;
    }

    private function reportProgress(string $message, ?int $percent = null): void
    {
        $this->logMessage($message);
        if ($this->progressCallback) {
            call_user_func($this->progressCallback, $message, $percent, $this->stats);
        }
    }

    /**
     * Analyze threads that need analysis
     *
     * @param int $batchSize Number of threads to analyze
     * @param int $delayMs Delay between API calls
     * @return array Statistics
     */
    public function analyzeBatch(int $batchSize = 10, int $delayMs = 1000): array
    {
        if (!$this->isConfigured()) {
            throw new Exception("Claude API key not configured");
        }

        $this->stats['start_time'] = microtime(true);
        $this->reportProgress("Starting thread analysis batch", 0);

        // Get threads that need analysis
        $threads = $this->getThreadsNeedingAnalysis($batchSize);

        if (empty($threads)) {
            $this->reportProgress("No threads need analysis", 100);
            return $this->stats;
        }

        $total = count($threads);
        $this->reportProgress("Found {$total} threads to analyze", 5);

        foreach ($threads as $index => $thread) {
            $percent = (int)((($index + 1) / $total) * 100);

            try {
                $this->reportProgress(
                    "Analyzing thread " . ($index + 1) . "/{$total}: " . substr($thread['subject'] ?? 'No subject', 0, 50),
                    $percent
                );

                $this->analyzeThread($thread['thread_id']);
                $this->stats['threads_analyzed']++;

                // Rate limiting
                if ($index < $total - 1 && $delayMs > 0) {
                    usleep($delayMs * 1000);
                }

            } catch (Exception $e) {
                $this->stats['threads_failed']++;
                $this->logMessage("Failed to analyze thread {$thread['thread_id']}: " . $e->getMessage(), 'error');
            }
        }

        $duration = round(microtime(true) - $this->stats['start_time'], 2);
        $this->reportProgress(
            "Batch complete in {$duration}s. Analyzed: {$this->stats['threads_analyzed']}, Failed: {$this->stats['threads_failed']}",
            100
        );

        return $this->stats;
    }

    /**
     * Get threads that need analysis
     *
     * @param int $limit
     * @return array
     */
    private function getThreadsNeedingAnalysis(int $limit): array
    {
        // Get threads that either:
        // 1. Have never been analyzed (not in thread_analysis)
        // 2. Need re-analysis (needs_reanalysis = 1)
        return $this->db->fetchAll(
            "SELECT
                e.thread_id,
                MIN(e.subject) as subject,
                MAX(e.date) as last_email_date,
                COUNT(*) as email_count,
                MAX(e.contact_id) as contact_id
             FROM emails e
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE e.thread_id IS NOT NULL
               AND e.thread_id != ''
               AND (ta.id IS NULL OR ta.needs_reanalysis = 1)
             GROUP BY e.thread_id
             HAVING email_count >= 1
             ORDER BY last_email_date DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Analyze a complete email thread
     *
     * @param string $threadId Gmail thread ID
     * @return array Analysis results
     */
    public function analyzeThread(string $threadId): array
    {
        // Get all emails in thread

        if (isset($_REQUEST['threadID'])){
            $extraSQL = " WHERE thread_id = '{$_REQUEST['threadID']}'";
            $emails = $this->db->fetchAll(
                "SELECT * FROM emails
                 {$extraSQL}
                 ORDER BY date ASC"
            );
        }else{
            $emails = $this->db->fetchAll(
                "SELECT * FROM emails
                 WHERE thread_id = ?
                 ORDER BY date ASC",
                [$threadId]
            );
        }
        //echo $threadId;
        //exit();

        if (empty($emails)) {
            throw new Exception("No emails found for thread: {$threadId}");
        }

        $this->stats['emails_processed'] += count($emails);

        // RAG: Retrieve similar past threads for context
        $ragContext = $this->retrieveSimilarThreads($emails);

        // Build thread context for Claude with RAG context
        $prompt = $this->buildThreadAnalysisPrompt($emails, $ragContext);

        //file_put_contents("logs/__ddbgA.log","\n emails: ".print_r($emails, true), FILE_APPEND);
        //file_put_contents("logs/__ddbgA.log","\n argContext: ".print_r($ragContext, true), FILE_APPEND);
        //echo 222;
        //print_r($this->db);
        //exit();
        // Call AI API (Claude or OpenAI)
        $this->logToDb('info', 'ThreadAnalyzer', "Starting analysis for thread {$threadId} with " . count($emails) . " emails", [
            'thread_id' => $threadId,
            'email_count' => count($emails),
            'provider' => $this->config['provider'] ?? 'claude'
        ], $threadId);

        $response = $this->callClaudeAPI($prompt, $threadId);
        //print_r($response);
        //exit();
        file_put_contents("../logs/__ddbgA.log","\n prompt: ".print_r($prompt, true), FILE_APPEND);
        file_put_contents("../logs/__ddbgA.log","\n resp: ".print_r($response, true), FILE_APPEND);
        //exit();
        // Parse response
        $analysis = $this->parseAnalysisResponse($response);

        // Store analysis
        $this->storeThreadAnalysis($threadId, $emails, $analysis, $response);

        // Extract and store quotes
        if (!empty($analysis['quotes'])) {
            $this->stats['quotes_extracted'] += count($analysis['quotes']);
            $this->storeQuotes($emails, $analysis['quotes']);
        }

        // Count inconsistencies
        if (!empty($analysis['inconsistencies'])) {
            $this->stats['inconsistencies_found'] += count($analysis['inconsistencies']);
        }

        // Mark all emails in thread as analyzed
        $this->db->execute(
            "UPDATE emails SET is_analyzed = 1 WHERE thread_id = ?",
            [$threadId]
        );

        return $analysis;
    }

    /**
     * Retrieve similar past threads for RAG context
     *
     * @param array $emails Current thread emails
     * @return array Similar threads with their analyses
     */
    private function retrieveSimilarThreads(array $emails): array
    {
        try {
            // Build search query from thread content
            $searchQuery = $this->buildThreadSearchQuery($emails);

            // Search vector memory for similar threads
            $vectorMemory = new VectorMemory();
            $similarResults = $vectorMemory->search($searchQuery, 3, ['type' => 'thread_summary']);

            $similarThreads = [];
            foreach ($similarResults as $result) {
                // Get thread analysis from database
                $threadId = $result['metadata']['thread_id'] ?? null;
                if ($threadId) {
                    $analysis = $this->db->fetchOne(
                        "SELECT ta.*, c.name as contact_name
                         FROM thread_analysis ta
                         LEFT JOIN contacts c ON ta.contact_id = c.id
                         WHERE ta.thread_id = ?",
                        [$threadId]
                    );

                    if ($analysis) {
                        $similarThreads[] = [
                            'similarity' => $result['similarity'] ?? 0,
                            'thread_id' => $threadId,
                            'contact_name' => $analysis['contact_name'],
                            'summary' => $result['content'] ?? '',
                            'deception_score' => $result['metadata']['deception_score'] ?? null,
                            'trust_score' => $result['metadata']['trust_score'] ?? null,
                            'risk_level' => $result['metadata']['risk_level'] ?? null
                        ];
                    }
                }
            }

            return $similarThreads;

        } catch (Exception $e) {
            error_log("RAG retrieval failed: " . $e->getMessage());
            return []; // Continue without RAG context if it fails
        }
    }

    /**
     * Build search query from thread emails
     *
     * @param array $emails Thread emails
     * @return string Search query
     */
    private function buildThreadSearchQuery(array $emails): string
    {
        // Extract key topics from thread
        $subjects = array_unique(array_column($emails, 'subject'));
        $mainSubject = $subjects[0] ?? 'discussion';

        // Get snippet from first and last email
        $firstBody = strip_tags($emails[0]['body_text'] ?? $emails[0]['body_html'] ?? '');
        $lastBody = strip_tags($emails[count($emails) - 1]['body_text'] ?? '');

        $snippet = substr($firstBody, 0, 200) . ' ... ' . substr($lastBody, 0, 200);

        return "Thread about: {$mainSubject}. {$snippet}";
    }

    /**
     * Build analysis prompt for complete thread
     *
     * @param array $emails All emails in thread
     * @param array $ragContext Similar past threads for context
     * @return string
     */
    private function buildThreadAnalysisPrompt(array $emails, array $ragContext = []): string
    {
        $emailCount = count($emails);
        $firstEmail = $emails[0];
        $lastEmail = $emails[$emailCount - 1];

        // Build conversation timeline
        $timeline = "";
        foreach ($emails as $index => $email) {
            $num = $index + 1;
            $direction = $email['direction'] === 'inbound' ? 'FROM THEM' : 'FROM US';
            $date = date('Y-m-d H:i', strtotime($email['date']));
            $subject = $email['subject'] ?? '(No Subject)';

            // Get email body and strip quoted content
            $body = $email['body_text'] ?? $email['body_html'] ?? '';
            $body = $this->stripQuotedContent($body);

            // Truncate long emails
            if (strlen($body) > 2000) {
                $body = substr($body, 0, 2000) . "\n[... email truncated ...]";
            }

            $timeline .= "━━━ EMAIL #{$num} ({$direction}) ━━━\n";
            $timeline .= "Date: {$date}\n";
            $timeline .= "Subject: {$subject}\n";
            $timeline .= "Content:\n{$body}\n\n";
        }

        // Build RAG context section
        $ragSection = "";
        if (!empty($ragContext)) {
            $ragSection = "\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
            $ragSection .= "SIMILAR PAST CONVERSATIONS (for context):\n";
            $ragSection .= "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n";
            $ragSection .= "We found " . count($ragContext) . " similar past situations. Use these as reference:\n\n";

            foreach ($ragContext as $i => $similar) {
                $num = $i + 1;
                $similarity = round($similar['similarity'] * 100, 1);
                $ragSection .= "SIMILAR #{$num} (Similarity: {$similarity}%):\n";
                $ragSection .= "Contact: " . ($similar['contact_name'] ?? 'Unknown') . "\n";
                $ragSection .= "Summary: " . substr($similar['summary'], 0, 300) . "\n";

                if ($similar['deception_score'] !== null) {
                    $ragSection .= "Deception Score: " . round($similar['deception_score'] * 100) . "%\n";
                }
                if ($similar['trust_score'] !== null) {
                    $ragSection .= "Trust Score: " . round($similar['trust_score'] * 100) . "%\n";
                }
                if ($similar['risk_level']) {
                    $ragSection .= "Risk Level: {$similar['risk_level']}\n";
                }
                $ragSection .= "\n";
            }

            $ragSection .= "Use the patterns from these similar situations to inform your analysis.\n";
            $ragSection .= "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n";
        }

        // Build participant list
        $participantStats = [];
        foreach ($emails as $email) {
            $from = $email['from_email'];
            $fromName = $email['from_name'] ?? '';
            if (!isset($participantStats[$from])) {
                $participantStats[$from] = [
                    'name' => $fromName,
                    'email_count' => 0,
                    'direction' => $email['direction'] === 'inbound' ? 'external' : 'self'
                ];
            }
            $participantStats[$from]['email_count']++;
        }

        $participantList = "";
        foreach ($participantStats as $email => $stats) {
            $role = $stats['direction'] === 'self' ? '(US)' : '(THEM)';
            $participantList .= "- {$stats['name']} <{$email}> {$role}: {$stats['email_count']} emails\n";
        }

        // Get owner email from config
        $ownerEmail = defined('GMAIL_USER_EMAIL') ? GMAIL_USER_EMAIL : 'dan@danmarcrm.com';
        $ownerPersona = defined('AI_PERSONA') ? AI_PERSONA : 'Dan runs DanmarCRM for 22 years, developing CRM software and integrations.';

        return <<<PROMPT
You are a behavioral analyst, business strategist, and protective advisor for Dan.

OWNER CONTEXT:
- Owner email: {$ownerEmail}
- {$ownerPersona}
- Dan is the GOOD GUY who must be protected. Identify any threats, manipulation, or harmful patterns from clients.
- Your job: Analyze emails, suggest business opportunities, generate reply options, and protect Dan's interests.

THREAD INFO:
- Total emails: {$emailCount}
- First email: {$firstEmail['date']}
- Last email: {$lastEmail['date']}
- Thread duration: From {$firstEmail['date']} to {$lastEmail['date']}

PARTICIPANTS IN THIS CONVERSATION:
{$participantList}
{$ragSection}
CONVERSATION TIMELINE:
{$timeline}

Analyze this ENTIRE conversation and respond with JSON:

{
  "thread_summary": "<2-3 sentence summary of what this conversation is about and its outcome>",

  "participants_analysis": [
    {
      "email": "<participant's email address>",
      "name": "<participant's name>",
      "role": "<external|self - external=client/supplier, self=us>",
      "email_count": <number of emails they sent>,

      "psychological_profile": {
        "trust_score": <float 0.0-1.0, how trustworthy this person appears>,
        "deception_score": <float 0.0-1.0, likelihood they are being deceptive>,
        "manipulation_score": <float 0.0-1.0, how manipulative their communication is>,
        "reliability_score": <float 0.0-1.0, do they follow through on commitments>,
        "aggression_score": <float 0.0-1.0, level of aggressive/demanding behavior>
      },

      "behavioral_indicators": {
        "communication_style": "<direct|evasive|assertive|passive-aggressive|professional|casual>",
        "consistency": "<consistent|inconsistent|contradictory> - do they change their story?",
        "transparency": "<open|guarded|secretive|deceptive> - how forthcoming are they?",
        "emotional_state": "<calm|anxious|frustrated|angry|desperate|confident>",
        "negotiation_tactics": ["<tactic1>", "<tactic2>"],
        "pressure_tactics": ["<any pressure/manipulation tactics used>"]
      },

      "red_flags": [
        {
          "flag": "<description of concerning behavior>",
          "severity": "<low|medium|high|critical>",
          "evidence": "<quote or reference from their emails>"
        }
      ],

      "positive_signals": [
        {
          "signal": "<positive behavior observed>",
          "evidence": "<quote or reference>"
        }
      ],

      "lies_or_inconsistencies": [
        {
          "claim": "<what they claimed>",
          "contradiction": "<how it contradicts other statements or facts>",
          "emails_involved": "<email numbers where this appears>"
        }
      ],

      "overall_assessment": "<1-2 sentence assessment of this person's trustworthiness and intentions>",
      "risk_level": "<low|medium|high|critical>",
      "verdict": "<trustworthy|cautious|suspicious|untrustworthy>"
    }
  ],

  "sentiment_analysis": {
    "overall_sentiment": "<positive|neutral|negative>",
    "sentiment_score": <float -1.0 to 1.0>,
    "sentiment_trend": "<improving|stable|declining> - how sentiment changed from start to end",
    "tone_progression": "<how the tone evolved through the conversation>"
  },

  "key_points": [
    "<main point 1>",
    "<main point 2>",
    "<main point 3>"
  ],

  "negotiation_analysis": {
    "type": "<price_negotiation|service_request|support|information|complaint|other>",
    "their_stance": "<aggressive|collaborative|passive|demanding|flexible>",
    "our_stance": "<accommodating|firm|flexible|defensive>",
    "outcome": "<resolved|ongoing|abandoned|escalated>",
    "who_initiated": "<them|us>",
    "resolution_summary": "<how it was resolved or current status>"
  },

  "quotes_and_pricing": [
    {
      "email_number": <which email it appeared in>,
      "amount": <number>,
      "currency": "<EUR|USD|etc>",
      "description": "<what it's for>",
      "direction": "<sent_by_us|received_from_them>",
      "type": "<initial|counter|final|discount_request>",
      "context": "<why this quote was given>"
    }
  ],

  "commitments_made": [
    {
      "by": "<them|us>",
      "commitment": "<what was promised>",
      "email_number": <which email>,
      "deadline": "<if mentioned>",
      "was_fulfilled": "<yes|no|unknown|pending> - look if later emails show it was done"
    }
  ],

  "inconsistencies": [
    {
      "type": "<contradiction|changed_story|avoided_question|false_claim>",
      "description": "<what the inconsistency is>",
      "evidence": "<compare email #X where they said A vs email #Y where they said B>",
      "severity": "<low|medium|high>"
    }
  ],

  "deception_indicators": {
    "deception_score": <float 0.0 to 1.0>,
    "trust_score": <float 0.0 to 1.0>,
    "shady_behavior_score": <float 0.0 to 1.0>,
    "red_flags": ["<flag1>", "<flag2>"],
    "hiding_information": "<evidence of withholding relevant information>",
    "manipulation_tactics": ["<any attempts to trick, deceive, or manipulate>"],
    "reasoning": "<why you assigned these scores>"
  },

  "business_risks": {
    "payment_risk": {
      "score": <float 0.0 to 1.0>,
      "payment_delays": "<evidence of delaying or avoiding payments>",
      "excuses_used": ["<list of excuses given for non-payment or delays>"],
      "payment_history_issues": "<any mentioned payment problems>"
    },
    "project_theft_risk": {
      "score": <float 0.0 to 1.0>,
      "knowledge_extraction": "<attempts to extract info to give to competitors>",
      "competitor_mentions": ["<any mentions of other companies doing similar work>"],
      "suspicious_questions": ["<questions that suggest they want to replicate our work elsewhere>"]
    },
    "relationship_risks": [
      {
        "type": "<delay_tactics|scope_creep|blame_shifting|ghosting|false_promises>",
        "description": "<what happened>",
        "evidence": "<email numbers and quotes>"
      }
    ]
  },

  "entities_mentioned": {
    "amounts": [{"value": <num>, "currency": "<curr>", "context": "<what for>"}],
    "dates": [{"date": "<date>", "context": "<what for>"}],
    "products_services": ["<item1>", "<item2>"],
    "companies": ["<company1>"],
    "people": ["<person1>"]
  },

  "action_items": [
    {
      "for": "<them|us>",
      "action": "<what needs to be done>",
      "priority": "<high|medium|low>",
      "deadline": "<if mentioned>"
    }
  ],

  "psychological_insights": {
    "communication_style": "<direct|indirect|formal|casual|aggressive|passive>",
    "decision_making": "<fast|slow|indecisive|confident>",
    "price_sensitivity": "<very_high|high|medium|low> - based on their reactions to quotes",
    "negotiation_skill": "<novice|average|skilled|expert>",
    "reliability_indicators": "<do they follow through? do they change their story?>",
    "pressure_tactics": ["<any tactics they used>"]
  },

  "recommendations": {
    "next_steps": ["<what we should do next>"],
    "how_to_approach": "<advice on how to handle this client/supplier>",
    "pricing_strategy": "<should we quote higher/lower, any insights on their budget>",
    "warnings": ["<things to watch out for with this contact>"],
    "overall_risk_level": "<low|medium|high|critical>",
    "client_verdict": "<trustworthy|cautious|risky|avoid>"
  },

  "reply_suggestions": [
    {
      "label": "<short label like 'Agree & Proceed' or 'Ask Clarification'>",
      "tone": "<positive|neutral|firm|friendly|professional>",
      "preview": "<first 60 chars of reply>",
      "full_text": "<complete ready-to-send email reply with greeting and sign-off>"
    }
  ],

  "business_opportunities": {
    "cross_sell_products": [
      {
        "product_idea": "<what product/service we could offer based on their needs mentioned in emails>",
        "why": "<evidence from emails showing they need this>",
        "how_to_pitch": "<suggested approach to bring this up>"
      }
    ],
    "software_development_ideas": [
      {
        "idea": "<custom software we could build for them based on their problems/needs>",
        "problem_they_have": "<what problem from the emails this solves>",
        "scope_estimate": "<small|medium|large>",
        "pitch_approach": "<how to suggest this to them>"
      }
    ],
    "quote_analysis": {
      "is_quote_related": "<true if this thread is about pricing/quotes, false otherwise>",
      "current_situation": "<summary of quote status from emails>",
      "recommended_action": "<what to propose - accept/counter/hold>",
      "suggested_price_adjustment": "<higher/lower/same and why>",
      "negotiation_leverage": "<what leverage do we have based on emails>",
      "closing_strategy": "<how to close this deal>"
    },
    "persuasion_strategy": {
      "client_motivation": "<what motivates this client based on their emails>",
      "pain_points": ["<pain point 1 from emails>", "<pain point 2>"],
      "questions_to_ask": [
        {
          "question": "<strategic question to ask them>",
          "why_ask": "<what info this reveals>",
          "expected_answers": ["<possible answer 1>", "<possible answer 2>"],
          "follow_up_actions": ["<what to do based on answer 1>", "<what to do based on answer 2>"]
        }
      ],
      "objections_to_expect": ["<likely objection based on emails>"],
      "rebuttals": ["<how to counter each objection>"],
      "buying_signals": ["<positive signals from emails indicating interest>"],
      "recommended_approach": "<overall strategy to win them over>"
    }
  },

  "protective_analysis": {
    "threats_to_owner": [
      {
        "threat_type": "<payment_risk|scope_creep|manipulation|knowledge_theft|reputation_risk|time_waste>",
        "description": "<what the threat is>",
        "severity": "<low|medium|high|critical>",
        "evidence": "<quote from emails showing this threat>",
        "recommended_action": "<how to protect against this>"
      }
    ],
    "harmful_patterns": [
      {
        "pattern": "<pattern name like 'Playing vendors against each other' or 'Endless revision requests'>",
        "evidence": "<proof from emails>",
        "risk_to_dan": "<how this could hurt Dan>"
      }
    ],
    "grammar_analysis": [
      {
        "participant_email": "<email address>",
        "participant_name": "<name>",
        "spelling_errors": "<count of spelling mistakes noticed>",
        "grammar_errors": "<count of grammar issues>",
        "professionalism_score": "<float 0.0-1.0, how professional their writing is>",
        "writing_quality": "<poor|average|good|excellent>",
        "notes": "<observations about their writing style>"
      }
    ],
    "overall_safety_score": "<float 0.0-1.0, how safe is this client for Dan>",
    "protection_alerts": ["<alert 1 if any serious concerns>", "<alert 2>"]
  },

  "churn_risk_signals": {
    "exit_intent_score": <float 0.0-1.0, likelihood they are planning to leave>,
    "exit_signals": [
      {
        "signal": "<what they said or did suggesting they might leave>",
        "severity": "<low|medium|high|critical>",
        "evidence": "<exact quote from email>",
        "email_number": <which email>
      }
    ],
    "competitor_mentions": [
      {
        "competitor": "<name of competitor or alternative mentioned>",
        "context": "<how they mentioned it - comparing, considering, already using?>",
        "threat_level": "<curious|comparing|considering|decided>"
      }
    ],
    "disengagement_signals": {
      "response_slowing": "<true|false - are their responses getting slower?>",
      "enthusiasm_declining": "<true|false - less enthusiastic than before?>",
      "communication_frequency_drop": "<true|false - contacting less often?>",
      "short_replies": "<true|false - replies getting shorter/curt?>"
    },
    "goodbye_language": {
      "detected": <true|false>,
      "examples": ["<farewell phrases used>"],
      "context": "<was it project end, relationship end, or just email sign-off?>"
    },
    "contract_end_signals": {
      "mentioned_ending": <true|false>,
      "mentioned_not_renewing": <true|false>,
      "asked_about_data_export": <true|false>,
      "requested_documentation": <true|false - requesting docs before leaving?>,
      "mentioned_budget_cuts": <true|false>,
      "mentioned_change_in_direction": <true|false>
    },
    "retention_opportunities": ["<what could we do to keep them?>"],
    "churn_risk_summary": "<1-2 sentence summary of churn risk>",
    "recommended_retention_action": "<specific action to prevent churn>"
  },

  "price_perception_analysis": {
    "quote_reaction_type": "<immediate_accept|accept|negotiate|hesitate|reject|shock|ignore|not_applicable>",
    "perceived_value": "<bargain|fair|acceptable|high|very_high|outrageous|unknown>",
    "price_sensitivity_score": <float 0.0-1.0, how sensitive to pricing they are>,
    "willingness_to_pay_more": <float 0.0-1.0, signals they would accept higher prices>,
    "negotiation_aggressiveness": "<passive|mild|moderate|aggressive|very_aggressive>",
    "price_objections": [
      {
        "objection": "<what they said about price>",
        "severity": "<mild|moderate|strong>",
        "email_number": <which email>,
        "response_strategy": "<how to address this objection>"
      }
    ],
    "budget_signals": {
      "budget_mentioned": <true|false>,
      "budget_constraints": "<what constraints they mentioned if any>",
      "budget_flexibility": "<tight|moderate|flexible|generous|unknown>",
      "approval_process_mentioned": <true|false - do they need approval?>
    },
    "value_perception": {
      "understands_value": <true|false>,
      "questioned_value": <true|false>,
      "compared_to_competitors": <true|false>,
      "asked_for_justification": <true|false>
    },
    "discount_behavior": {
      "requested_discount": <true|false>,
      "discount_percent_requested": <number or null>,
      "discount_justification": "<reason they gave for wanting discount>",
      "frequency_of_discount_requests": "<never|rare|occasional|frequent|always>"
    },
    "pricing_strategy_recommendation": "<recommend higher/lower/same pricing and why>",
    "maximum_they_would_pay": "<estimate based on signals, or 'unknown'>"
  },

  "support_satisfaction_analysis": {
    "satisfaction_score": <float 0.0-1.0, overall support satisfaction>,
    "frustration_level": "<none|mild|moderate|high|extreme>",
    "frustration_incidents": [
      {
        "issue": "<what frustrated them>",
        "severity": "<low|medium|high>",
        "our_response": "<how we handled it>",
        "resolution": "<resolved|unresolved|partial>",
        "email_number": <which email>
      }
    ],
    "complaint_patterns": {
      "frequency": "<never|rare|occasional|frequent|constant>",
      "types": ["<type of complaint 1>", "<type 2>"],
      "escalation_tendency": <float 0.0-1.0, likelihood to escalate>
    },
    "patience_level": "<very_patient|patient|neutral|impatient|very_impatient>",
    "appreciation_expressed": <true|false - did they thank us or express appreciation?>,
    "would_recommend_us": "<likely|unlikely|unknown>",
    "relationship_repair_needed": <true|false>,
    "repair_suggestions": ["<how to improve the relationship>"]
  },

  "deep_client_psychology": {
    "decision_maker_analysis": {
      "decision_maker_type": "<primary|influencer|gatekeeper|user|unknown>",
      "authority_level": "<final_authority|needs_approval|recommender|no_authority|unknown>",
      "reports_to_whom": "<if mentioned, who do they need to convince?>",
      "decision_speed": "<impulsive|fast|moderate|slow|very_slow>",
      "decision_criteria": ["<what factors drive their decisions>"]
    },
    "personality_traits": ["<trait1>", "<trait2>", "<trait3>"],
    "communication_preferences": {
      "preferred_channel": "<email|phone|video|in_person|mixed>",
      "response_urgency_expected": "<immediate|same_day|next_day|within_week|flexible>",
      "detail_orientation": "<big_picture|balanced|detail_oriented|extremely_detailed>",
      "emotional_vs_rational": "<very_emotional|emotional|balanced|rational|very_rational>",
      "formality_preference": "<very_casual|casual|neutral|formal|very_formal>"
    },
    "motivators": ["<what motivates them - efficiency, cost savings, innovation, etc>"],
    "fears_and_concerns": ["<what worries them>"],
    "trust_building_opportunities": ["<how to build more trust>"],
    "loyalty_indicators": {
      "loyalty_score": <float 0.0-1.0>,
      "long_term_potential": "<low|medium|high>",
      "referral_likelihood": <float 0.0-1.0>,
      "relationship_investment_level": "<minimal|low|moderate|high|very_high>"
    }
  },

  "software_opportunity_mining": {
    "pain_points_detected": [
      {
        "pain_point": "<specific problem they mentioned>",
        "evidence": "<quote from email>",
        "frequency_mentioned": <how many times mentioned>,
        "urgency": "<low|medium|high|critical>",
        "current_solution": "<how they handle it now, if mentioned>"
      }
    ],
    "automation_opportunities": [
      {
        "process": "<manual process they do that could be automated>",
        "evidence": "<how you know this from emails>",
        "estimated_time_savings": "<low|medium|high>",
        "complexity": "<simple|moderate|complex>",
        "recommended_solution": "<what to build>"
      }
    ],
    "integration_needs": [
      {
        "system_a": "<one system they use>",
        "system_b": "<another system that needs connecting>",
        "evidence": "<how you know from emails>",
        "complexity": "<simple|moderate|complex>"
      }
    ],
    "data_problems": [
      {
        "problem": "<data issue they have - scattered, duplicates, manual entry, etc>",
        "evidence": "<from emails>",
        "solution_type": "<migration|cleanup|integration|dashboard>"
      }
    ],
    "industry_context": {
      "client_industry": "<their industry based on emails>",
      "industry_specific_needs": ["<common needs in this industry>"],
      "regulatory_requirements": ["<if any compliance needs mentioned>"]
    },
    "specific_software_proposals": [
      {
        "title": "<name of software/tool to propose>",
        "problem_solved": "<what it solves for them>",
        "scope": "<tiny|small|medium|large|enterprise>",
        "estimated_hours": "<range like 40-80>",
        "pitch_approach": "<how to bring this up>",
        "win_probability": <float 0.0-1.0>
      }
    ]
  }

}

CRITICAL INSTRUCTIONS:
- BE CONCISE. Keep all text fields SHORT (1-2 sentences max).
- Profile EACH participant separately in participants_analysis
- SKIP per_email_analysis - too verbose
- Only include 3-5 most important red_flags, key_points, action_items
- Only include quotes_and_pricing if actual prices are mentioned
- Look for deception, payment delays, manipulation, knowledge extraction attempts
- Omit empty arrays - if no red_flags found, use empty array []

REPLY SUGGESTIONS (reply_suggestions):
- Generate exactly 5 different reply options based on the conversation
- Each reply must be COMPLETE and ready to copy-paste and send
- Include proper greeting (Hi [name],) and sign-off (Best regards, Dan)
- Options should cover: positive response, clarifying question, boundary-setting, solution-focused, brief acknowledgment
- Base the replies on ACTUAL email content - reference specific things they mentioned

BUSINESS OPPORTUNITIES (business_opportunities):
- Cross-sell: Look at what they're discussing - what OTHER products/services could help them?
- Software ideas: What custom software could solve problems they mentioned in emails?
- Quote analysis: Only fill if this is about pricing/quotes, otherwise set is_quote_related to false
- Persuasion: What questions should Dan ask? What objections might come? How to close?
- ALL insights must come from the EMAIL CONTENT - not generic advice

PROTECTIVE ANALYSIS (protective_analysis):
- Dan (owner) is the GOOD GUY - protect his interests
- Look for: payment risks, scope creep, manipulation, knowledge theft, time wasters
- Grammar analysis: Count actual errors you see in their emails, assess professionalism
- Be SPECIFIC with evidence - quote the emails
- overall_safety_score: 1.0 = completely safe, 0.0 = avoid this client

CHURN RISK SIGNALS (churn_risk_signals) - CRITICAL FOR BUSINESS SURVIVAL:
- Look for ANY hint they might leave: competitor mentions, budget issues, dissatisfaction
- "Goodbye language" = farewell phrases beyond normal email sign-offs
- Disengagement = slower responses, shorter replies, less enthusiasm
- Contract signals = asking about data export, documentation, ending dates
- BE SENSITIVE to subtle signals - even small hints of leaving are valuable
- exit_intent_score: 0.0 = loyal, 1.0 = definitely leaving soon

PRICE PERCEPTION ANALYSIS (price_perception_analysis) - CRITICAL FOR PRICING STRATEGY:
- How did they react to quotes? Shock, acceptance, negotiation?
- Did they ask for discounts? How aggressively?
- Do they understand the value we provide?
- Could we charge MORE? (high willingness_to_pay_more = we're underpricing)
- What is their budget reality? Tight or flexible?
- Maximum they would pay = your best estimate based on all signals

SUPPORT SATISFACTION (support_satisfaction_analysis):
- Any frustration expressed? How severe?
- Did they complain? About what? How often?
- Did they escalate issues?
- Express appreciation or just demands?
- Would they recommend us to others?

DEEP CLIENT PSYCHOLOGY (deep_client_psychology):
- Are they the decision maker or need approval?
- What personality traits are evident?
- What motivates them? Fears?
- How loyal are they? Referral potential?

SOFTWARE OPPORTUNITY MINING (software_opportunity_mining) - REVENUE OPPORTUNITY:
- What PAIN POINTS did they mention? (manual processes, spreadsheet hell, data problems)
- What could be AUTOMATED for them?
- What systems need INTEGRATION?
- What SOFTWARE could Dan build to solve their problems?
- Be SPECIFIC - propose actual software with scope estimates
- This is how Dan finds new work to stay in business!

BE A SKEPTICAL INVESTIGATOR - examine each person's behavior critically.
Look for evidence of deception, manipulation, and hidden agendas in each participant.
Be objective and evidence-based but err on the side of caution.
Respond ONLY with the JSON, no additional text

PROMPT;
    }

    /**
     * Public method to ask Claude a question and get text response
     * Used by API endpoints for ask_ai and generate_reply features
     */
    public function askClaude(string $prompt, ?string $threadId = null): string
    {
        $response = $this->callClaudeAPI($prompt, $threadId);
        return $response['content'] ?? '';
    }

    /**
     * Call AI API (supports both Claude and OpenAI)
     */
    private function callClaudeAPI(string $prompt, ?string $threadId = null): array
    {
        try {
            // Clean UTF-8 in prompt to avoid json_encode errors
            $cleanPrompt = $this->cleanUtf8($prompt);

            $provider = $this->config['provider'] ?? 'claude';
            // Generate unique request ID to track duplicate calls
            $ddbg = debug_backtrace(2);
            $requestId = uniqid('req_', true);
            file_put_contents(__DIR__ . '/../logs/prompt.log', "\n\nDate: " . date('Y-m-d H:i:s') . " [RequestID: {$requestId}] [ThreadID: " . ($threadId ?? 'N/A') . "] - Prompt: " . $cleanPrompt . "\n", FILE_APPEND);
            file_put_contents(__DIR__ . '/../logs/prompt.log', "\n\nDebug Backtrace: " . print_r($ddbg, true) . "\n", FILE_APPEND);
            // Log the request
            $this->logToDb('info', 'AI-Request', "Sending request to {$provider} ({$this->config['model']})", [
                'provider' => $provider,
                'model' => $this->config['model'],
                'max_tokens' => $this->config['max_tokens'],
                'prompt_length' => strlen($cleanPrompt),
                'prompt_preview' => substr($cleanPrompt, 0, 500) . '...',
                'prompt' => $cleanPrompt  // Full prompt sent to AI
            ], $threadId);

            if ($provider === 'openai') {
                // OpenAI request format
                $requestBody = [
                    'model' => $this->config['model'],
                    'max_tokens' => $this->config['max_tokens'],
                    'messages' => [
                        ['role' => 'user', 'content' => $cleanPrompt]
                    ]
                ];
            } else {
                // Claude request format
                $requestBody = [
                    'model' => $this->config['model'],
                    'max_tokens' => $this->config['max_tokens'],
                    'messages' => [
                        ['role' => 'user', 'content' => $cleanPrompt]
                    ]
                ];
            }

            $jsonBody = $this->safeJsonEncode($requestBody);
            $startTime = microtime(true);

            $response = $this->http->post($this->config['endpoint'], [
                'headers' => [
                    'Content-Type' => 'application/json'
                ],
                'body' => $jsonBody
            ]);

            $elapsedTime = round(microtime(true) - $startTime, 2);
            $body = json_decode($response->getBody()->getContents(), true);

            // Parse response based on provider
            if ($provider === 'openai') {
                if (!isset($body['choices'][0]['message']['content'])) {
                    throw new Exception("Invalid OpenAI response structure: " . json_encode($body));
                }
                $content = $body['choices'][0]['message']['content'];
                $usage = $body['usage'] ?? [];
                if (isset($usage['total_tokens'])) {
                    $this->stats['tokens_used'] += $usage['total_tokens'];
                }
            } else {
                if (!isset($body['content'][0]['text'])) {
                    throw new Exception("Invalid Claude response structure: " . json_encode($body));
                }
                $content = $body['content'][0]['text'];
                $usage = $body['usage'] ?? [];
                if (isset($usage)) {
                    $this->stats['tokens_used'] += ($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0);
                }
            }

            // Log the response
            $this->logToDb('info', 'AI-Response', "Received response from {$provider} in {$elapsedTime}s", [
                'provider' => $provider,
                'model' => $body['model'] ?? $this->config['model'],
                'elapsed_seconds' => $elapsedTime,
                'usage' => $usage,
                'response_length' => strlen($content),
                'response_preview' => substr($content, 0, 500) . '...',
                'response' => $content  // Full response from AI
            ], $threadId);

            // Save full request/response to disk for debugging
            $this->saveAnalysisLog($threadId, $cleanPrompt, $content, $usage, $elapsedTime, $provider);

            return [
                'content' => $content,
                'model' => $body['model'] ?? $this->config['model'],
                'usage' => $usage,
                'provider' => $provider
            ];

        } catch (Exception $e) {
            $this->logMessage("API request failed ({$this->config['provider']}): " . $e->getMessage(), 'error');
            $this->logToDb('error', 'AI-Error', "API request failed: " . $e->getMessage(), [
                'provider' => $this->config['provider'] ?? 'unknown',
                'error' => $e->getMessage()
            ], $threadId);
            throw new Exception("{$this->config['provider']} API error: " . $e->getMessage());
        }
    }

    /**
     * Parse JSON response from Claude
     */
    private function parseAnalysisResponse(array $response): array
    {
        $content = $response['content'];

        // Strip markdown code block wrappers if present
        $content = preg_replace('/^```(?:json)?\s*/i', '', $content);
        $content = preg_replace('/\s*```\s*$/', '', $content);
        $content = trim($content);

        // Detect AI refusal responses (content moderation)
        $refusalPatterns = [
            '/I\'m sorry.*can\'t assist/i',
            '/I cannot help with/i',
            '/I\'m not able to/i',
            '/I apologize.*cannot/i',
            '/I\'m unable to/i',
            '/This request.*violates/i',
            '/I can\'t provide/i',
            '/I cannot provide/i'
        ];

        foreach ($refusalPatterns as $pattern) {
            if (preg_match($pattern, $content)) {
                $this->logMessage("AI refused to analyze. Response: " . substr($content, 0, 500), 'warning');
                throw new Exception("AI declined to analyze this thread. This may happen with sensitive content. Try again or contact support.");
            }
        }

        $jsonStart = strpos($content, '{');
        $jsonEnd = strrpos($content, '}');

        if ($jsonStart === false || $jsonEnd === false) {
            $this->logMessage("No JSON found in response. Content: " . substr($content, 0, 500), 'error');
            throw new Exception("No JSON found in API response. The AI may have returned an unexpected response format.");
        }

        $jsonString = substr($content, $jsonStart, $jsonEnd - $jsonStart + 1);

        // Try to fix common JSON issues
        // Remove trailing commas before } or ]
        $jsonString = preg_replace('/,\s*([\}\]])/', '$1', $jsonString);

        $analysis = json_decode($jsonString, true);

        if (json_last_error() !== JSON_ERROR_NONE) {
            // Log the full JSON to file for debugging
            file_put_contents(__DIR__ . '/../logs/failed_json.log', "[" . date('Y-m-d H:i:s') . "]\n" . $jsonString . "\n\n", FILE_APPEND);
            $this->logMessage("JSON parse failed. Error: " . json_last_error_msg() . ". Full JSON logged to failed_json.log", 'error');
            throw new Exception("Failed to parse JSON: " . json_last_error_msg());
        }

        return $analysis;
    }

    /**
     * Store thread analysis results
     */
    private function storeThreadAnalysis(string $threadId, array $emails, array $analysis, array $rawResponse): void
    {
        $firstEmail = $emails[0];
        $lastEmail = $emails[count($emails) - 1];
        $sentiment = $analysis['sentiment_analysis'] ?? [];
        $negotiation = $analysis['negotiation_analysis'] ?? [];
        $deception = $analysis['deception_indicators'] ?? [];

        // Store in thread_analysis table with full analysis
        // Ensure summary is a string (Claude might return it as an array)
        $summary = $analysis['thread_summary'] ?? null;
        if (is_array($summary)) {
            $summary = implode(' ', $summary);
        }

        $this->db->insertOrUpdate('thread_analysis', [
            'thread_id' => $threadId,
            'contact_id' => $lastEmail['contact_id'] ?? null,
            'email_count' => count($emails),
            'last_email_date' => $lastEmail['date'],
            'last_email_id' => $lastEmail['id'],
            'is_analyzed' => 1,
            'needs_reanalysis' => 0,
            'analyzed_at' => date('Y-m-d H:i:s'),
            'summary' => $summary,
            'raw_analysis' => $this->safeJsonEncode($analysis),
            'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
            'deception_score' => $deception['deception_score'] ?? 0
        ], ['contact_id', 'email_count', 'last_email_date', 'last_email_id', 'is_analyzed', 'needs_reanalysis', 'analyzed_at', 'summary', 'raw_analysis', 'key_points', 'deception_score']);

        // Also update MongoDB to keep dashboards in sync
        try {
            $mongo = MongoDatabase::getInstance();
            $mongo->insertOrUpdate('thread_analysis', [
                'thread_id' => $threadId,
                'is_analyzed' => 1,
                'needs_reanalysis' => 0,
                'analyzed_at' => date('Y-m-d H:i:s'),
                'summary' => $summary,
                'raw_analysis' => $this->safeJsonEncode($analysis),
                'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
                'deception_score' => $deception['deception_score'] ?? 0,
                'contact_id' => $lastEmail['contact_id'] ?? null,
                'email_count' => count($emails),
                'last_email_date' => $lastEmail['date'],
                'last_email_id' => $lastEmail['id']
            ], ['thread_id']);
        } catch (Exception $mongoEx) {
            // Log but don't fail the analysis if MongoDB sync fails
            error_log("MongoDB sync failed for thread {$threadId}: " . $mongoEx->getMessage());
        }

        // Store detailed analysis in email_analysis (linked to last email)
        // Ensure string fields are not arrays
        $tone = $negotiation['their_stance'] ?? null;
        if (is_array($tone)) {
            $tone = implode(', ', $tone);
        }
        $sentimentLabel = $sentiment['overall_sentiment'] ?? null;
        if (is_array($sentimentLabel)) {
            $sentimentLabel = is_array($sentimentLabel) ? implode(', ', $sentimentLabel) : $sentimentLabel;
        }

        $this->db->insertOrUpdate('email_analysis', [
            'email_id' => $lastEmail['id'],
            'sentiment_score' => $sentiment['sentiment_score'] ?? null,
            'sentiment_label' => $sentimentLabel,
            'tone' => $tone,
            'urgency_level' => null,
            'key_entities' => isset($analysis['entities_mentioned']) ? $this->safeJsonEncode($analysis['entities_mentioned']) : null,
            'commitments' => isset($analysis['commitments_made']) ? $this->safeJsonEncode($analysis['commitments_made']) : null,
            'requests' => null,
            'action_items' => isset($analysis['action_items']) ? $this->safeJsonEncode($analysis['action_items']) : null,
            'deception_flags' => isset($analysis['inconsistencies']) ? $this->safeJsonEncode($analysis['inconsistencies']) : null,
            'deception_score' => $deception['deception_score'] ?? null,
            'contains_quote' => !empty($analysis['quotes_and_pricing']) ? 1 : 0,
            'quote_amounts' => isset($analysis['quotes_and_pricing']) ? $this->safeJsonEncode($analysis['quotes_and_pricing']) : null,
            'summary' => $summary,
            'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
            'raw_analysis' => $this->safeJsonEncode($analysis),
            'model_used' => $rawResponse['model'] ?? $this->config['model'],
            'tokens_used' => ($rawResponse['usage']['input_tokens'] ?? 0) + ($rawResponse['usage']['output_tokens'] ?? 0),
            'analyzed_at' => date('Y-m-d H:i:s')
        ], ['email_id']);

        // Store all participants with their analysis
        $this->storeThreadParticipants($threadId, $analysis);

        // Log success to DB
        $this->logToDb('info', 'Thread analysis completed', $threadId, [
            'email_count' => count($emails),
            'participants' => count($analysis['participants_analysis'] ?? []),
            'deception_score' => $deception['deception_score'] ?? 0
        ]);
    }

    /**
     * Store extracted quotes
     */
    private function storeQuotes(array $emails, array $quotes): void
    {
        foreach ($quotes as $quote) {
            if (empty($quote['amount'])) continue;

            // Find which email in thread this quote came from
            $emailIndex = ($quote['email_number'] ?? 1) - 1;
            $email = $emails[$emailIndex] ?? $emails[count($emails) - 1];

            try {
                $this->db->insert('quotes', [
                    'contact_id' => $email['contact_id'],
                    'email_id' => $email['id'],
                    'amount' => $quote['amount'],
                    'currency' => $quote['currency'] ?? 'EUR',
                    'description' => $quote['description'] ?? $quote['context'] ?? null,
                    'direction' => $quote['direction'] === 'sent_by_us' ? 'sent' : 'received',
                    'quote_type' => $quote['type'] ?? 'initial',
                    'outcome' => 'pending',
                    'confidence_score' => 0.8,
                    'quote_date' => $email['date']
                ]);
            } catch (Exception $e) {
                $this->logMessage("Failed to store quote: " . $e->getMessage(), 'warning');
            }
        }
    }

    /**
     * Store thread participants (all contacts in the thread with their analysis)
     */
    private function storeThreadParticipants(string $threadId, array $analysis): void
    {
        $participants = $analysis['participants_analysis'] ?? [];

        foreach ($participants as $participant) {
            $email = $participant['email'] ?? null;
            if (!$email) continue;

            // Find or create contact
            $contact = $this->db->fetchOne("SELECT id FROM contacts WHERE email = ?", [$email]);
            if (!$contact) {
                // Create new contact
                $this->db->insert('contacts', [
                    'email' => $email,
                    'name' => $participant['name'] ?? null,
                    'type' => $participant['role'] === 'self' ? 'internal' : 'client',
                    'created_at' => date('Y-m-d H:i:s')
                ]);
                $contactId = $this->db->lastInsertId();
            } else {
                $contactId = $contact['id'];
            }

            $profile = $participant['psychological_profile'] ?? [];
            $behavioral = $participant['behavioral_indicators'] ?? [];

            // Store participant in thread_participants table
            try {
                $this->db->insertOrUpdate('thread_participants', [
                    'thread_id' => $threadId,
                    'contact_id' => $contactId,
                    'email_count' => $participant['email_count'] ?? 0,
                    'role' => $participant['role'] === 'self' ? 'self' : 'external',
                    'trust_score' => $profile['trust_score'] ?? null,
                    'deception_score' => $profile['deception_score'] ?? null,
                    'manipulation_score' => $profile['manipulation_score'] ?? null,
                    'reliability_score' => $profile['reliability_score'] ?? null,
                    'risk_level' => $participant['risk_level'] ?? null,
                    'verdict' => $participant['verdict'] ?? null,
                    'psychological_profile' => $this->safeJsonEncode($profile),
                    'behavioral_indicators' => $this->safeJsonEncode($behavioral),
                    'red_flags' => $this->safeJsonEncode($participant['red_flags'] ?? []),
                    'positive_signals' => $this->safeJsonEncode($participant['positive_signals'] ?? []),
                    'overall_assessment' => $participant['overall_assessment'] ?? null
                ], ['email_count', 'role', 'trust_score', 'deception_score', 'manipulation_score',
                    'reliability_score', 'risk_level', 'verdict', 'psychological_profile',
                    'behavioral_indicators', 'red_flags', 'positive_signals', 'overall_assessment']);

                // Store learned behaviors from this participant
                $this->storeLearnedBehaviors($contactId, $threadId, $participant);

            } catch (Exception $e) {
                $this->logMessage("Failed to store participant {$email}: " . $e->getMessage(), 'warning');
            }
        }
    }

    /**
     * Store learned behaviors for RAG context in future analyses
     */
    private function storeLearnedBehaviors(int $contactId, string $threadId, array $participant): void
    {
        // Store red flags as learned behaviors
        $redFlags = $participant['red_flags'] ?? [];
        foreach ($redFlags as $flag) {
            $this->addLearnedBehavior($contactId, $threadId, 'red_flag', $flag['flag'] ?? '', $flag['evidence'] ?? '', $flag['severity'] ?? 'medium');
        }

        // Store positive signals
        $positives = $participant['positive_signals'] ?? [];
        foreach ($positives as $signal) {
            $this->addLearnedBehavior($contactId, $threadId, 'positive_signal', $signal['signal'] ?? '', $signal['evidence'] ?? '', 'low');
        }

        // Store negotiation tactics
        $tactics = $participant['behavioral_indicators']['negotiation_tactics'] ?? [];
        foreach ($tactics as $tactic) {
            if (is_string($tactic)) {
                $this->addLearnedBehavior($contactId, $threadId, 'negotiation_tactic', $tactic, '', 'medium');
            }
        }

        // Store pressure tactics as potential deception indicators
        $pressure = $participant['behavioral_indicators']['pressure_tactics'] ?? [];
        foreach ($pressure as $tactic) {
            if (is_string($tactic)) {
                $this->addLearnedBehavior($contactId, $threadId, 'deception_indicator', $tactic, '', 'medium');
            }
        }

        // Store communication pattern
        $style = $participant['behavioral_indicators']['communication_style'] ?? null;
        if ($style) {
            $this->addLearnedBehavior($contactId, $threadId, 'communication_pattern', $style, '', 'low');
        }
    }

    /**
     * Add or update a learned behavior
     */
    private function addLearnedBehavior(int $contactId, string $threadId, string $type, string $description, string $evidence, string $severity): void
    {
        if (empty($description)) return;

        // Check if similar behavior already exists
        $existing = $this->db->fetchOne(
            "SELECT id, frequency FROM learned_behaviors WHERE contact_id = ? AND behavior_type = ? AND description = ?",
            [$contactId, $type, $description]
        );

        if ($existing) {
            // Update frequency and last seen
            $this->db->execute(
                "UPDATE learned_behaviors SET frequency = frequency + 1, last_seen_thread = ?, updated_at = NOW() WHERE id = ?",
                [$threadId, $existing['id']]
            );
        } else {
            // Insert new behavior
            try {
                $this->db->insert('learned_behaviors', [
                    'contact_id' => $contactId,
                    'behavior_type' => $type,
                    'description' => $description,
                    'evidence' => $evidence,
                    'severity' => $severity,
                    'frequency' => 1,
                    'first_seen_thread' => $threadId,
                    'last_seen_thread' => $threadId,
                    'confidence_score' => 0.8,
                    'is_active' => 1
                ]);
            } catch (Exception $e) {
                // Ignore duplicate errors
            }
        }
    }

    /**
     * Log to api_logs table for extension visibility
     */
    public function logToDb(string $level, string $source, string $message, array $context = [], ?string $threadId = null): void
    {
        try {
            $this->db->insert('api_logs', [
                'log_level' => $level,
                'source' => $source,
                'message' => $message,
                'context' => !empty($context) ? $this->safeJsonEncode($context) : null,
                'thread_id' => $threadId,
                'created_at' => date('Y-m-d H:i:s')
            ]);
        } catch (Exception $e) {
            // Fallback to file log
            $this->logMessage("DB log failed: " . $e->getMessage(), 'warning');
        }
    }

    /**
     * Mark thread for re-analysis (when new emails arrive)
     *
     * @param string $threadId
     */
    public function markThreadForReanalysis(string $threadId): void
    {
        $this->db->execute(
            "UPDATE thread_analysis SET needs_reanalysis = 1 WHERE thread_id = ?",
            [$threadId]
        );
    }

    /**
     * Get statistics
     */
    public function getStats(): array
    {
        return $this->stats;
    }

    private function logMessage(string $message, string $level = 'info'): void
    {
        $logFile = defined('LOG_PATH') ? LOG_PATH . 'thread_analyzer.log' : __DIR__ . '/../logs/thread_analyzer.log';
        $timestamp = date('Y-m-d H:i:s');
        $levelUpper = strtoupper($level);
        file_put_contents($logFile, "[{$timestamp}] [{$levelUpper}] {$message}\n", FILE_APPEND);

        if (php_sapi_name() === 'cli') {
            echo "[{$levelUpper}] {$message}\n";
        }
    }

    /**
     * Safely encode data to JSON, fixing UTF-8 issues
     */
    private function safeJsonEncode($data): string
    {
        $cleaned = $this->cleanUtf8($data);
        $json = json_encode($cleaned, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE);
        if ($json === false) {
            $this->logMessage("json_encode error: " . json_last_error_msg(), 'error');
            return '{}';
        }
        return $json;
    }

    /**
     * Recursively clean UTF-8 strings in data
     */
    private function cleanUtf8($data)
    {
        if (is_string($data)) {
            // Remove invalid UTF-8 sequences
            $data = mb_convert_encoding($data, 'UTF-8', 'UTF-8');
            // Remove null bytes and other problematic characters
            $data = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $data);
            return $data;
        }
        if (is_array($data)) {
            return array_map([$this, 'cleanUtf8'], $data);
        }
        return $data;
    }

    /**
     * Save full AI request/response to disk for debugging and review
     * Files are saved in logs/ai_analysis/ organized by date
     */
    private function saveAnalysisLog(?string $threadId, string $prompt, string $response, array $usage, float $elapsedTime, string $provider): void
    {
        try {
            $logDir = dirname(__DIR__) . '/logs/ai_analysis/' . date('Y-m-d');
            if (!is_dir($logDir)) {
                mkdir($logDir, 0755, true);
            }

            $timestamp = date('H-i-s');
            $safeThreadId = preg_replace('/[^a-zA-Z0-9_-]/', '_', $threadId ?? 'unknown');
            $filename = "{$logDir}/{$timestamp}_{$safeThreadId}.txt";

            $tokenInfo = "Input tokens: " . ($usage['input_tokens'] ?? 'N/A') .
                        ", Output tokens: " . ($usage['output_tokens'] ?? 'N/A') .
                        ", Total: " . (($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0));

            $logContent = "=" . str_repeat("=", 79) . "\n";
            $logContent .= "THREAD ANALYSIS LOG\n";
            $logContent .= "=" . str_repeat("=", 79) . "\n\n";
            $logContent .= "Thread ID: {$threadId}\n";
            $logContent .= "Timestamp: " . date('Y-m-d H:i:s') . "\n";
            $logContent .= "Provider: {$provider}\n";
            $logContent .= "Elapsed Time: {$elapsedTime}s\n";
            $logContent .= "Token Usage: {$tokenInfo}\n";
            $logContent .= "Prompt Length: " . strlen($prompt) . " chars\n";
            $logContent .= "Response Length: " . strlen($response) . " chars\n";
            $logContent .= "\n" . str_repeat("-", 80) . "\n";
            $logContent .= "PROMPT SENT TO AI:\n";
            $logContent .= str_repeat("-", 80) . "\n\n";
            $logContent .= $prompt;
            $logContent .= "\n\n" . str_repeat("-", 80) . "\n";
            $logContent .= "AI RESPONSE:\n";
            $logContent .= str_repeat("-", 80) . "\n\n";
            $logContent .= $response;
            $logContent .= "\n\n" . str_repeat("=", 80) . "\n";
            $logContent .= "END OF LOG\n";
            $logContent .= str_repeat("=", 80) . "\n";

            file_put_contents($filename, $logContent);

            // Also keep a summary log for quick reference
            $summaryFile = dirname(__DIR__) . '/logs/ai_analysis/summary.log';
            $summaryLine = date('Y-m-d H:i:s') . " | Thread: {$threadId} | Provider: {$provider} | Time: {$elapsedTime}s | Tokens: " . (($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0)) . " | File: {$filename}\n";
            file_put_contents($summaryFile, $summaryLine, FILE_APPEND);

        } catch (Exception $e) {
            // Don't fail the analysis if logging fails
            error_log("Failed to save analysis log: " . $e->getMessage());
        }
    }

    /**
     * Strip quoted content from email body to avoid duplication
     * Removes:
     * - Lines starting with > (quoted replies)
     * - "On X date, Y wrote:" patterns and everything after
     * - "From: ... Sent: ... To: ... Subject: ..." forwarded headers
     */
    private function stripQuotedContent(string $body): string
    {
        // Remove lines starting with > (quoted text)
        $body = preg_replace('/^>+.*$/m', '', $body);

        // Remove "On [date], [person] wrote:" and everything after
        $patterns = [
            '/On\s+.{10,60}\s+wrote:\s*$/mi',  // "On Monday, Jan 1, John wrote:"
            '/On\s+\d{1,2}[\/-]\d{1,2}[\/-]\d{2,4}.+wrote:\s*$/mi',  // "On 01/02/2023, John wrote:"
            '/-{2,}.*Original Message.*-{2,}/i',  // "-----Original Message-----"
            '/From:\s+.+\nSent:\s+.+\nTo:\s+.+\nSubject:\s+.+/i',  // Outlook forwarded header
            '/_{5,}.*$/m',  // _____ separator lines
        ];

        foreach ($patterns as $pattern) {
            if (preg_match($pattern, $body, $matches, PREG_OFFSET_CAPTURE)) {
                // Cut everything from the match onwards
                $body = substr($body, 0, $matches[0][1]);
            }
        }

        // Clean up multiple consecutive newlines
        $body = preg_replace('/\n{3,}/', "\n\n", $body);

        return trim($body);
    }
}

-            $body = $this->stripQuotedContent($body);

+            $body = self::stripQuotedContent($body);


Edit

private function stripQuotedContent(string $body): string

public static function stripQuotedContent(string $body): string