The file /var/www/mssql.danmarcrm.com/dev1/extension2/src/ThreadAnalyzer.php has been updated successfully.

<?php
/**
 * Thread Analyzer - Analyzes Complete Email Threads
 *
 * Unlike AIAnalyzer which processes individual emails,
 * this analyzes ENTIRE email threads (conversations) together.
 *
 * WHY THREADS?
 * - Detects inconsistencies across multiple emails
 * - Understands negotiation flow (quote → counter → agreement)
 * - Sees how tone/sentiment changes through conversation
 * - Better psychological profiling with full context
 *
 * CALLING MAP:
 * - Called by: cli/analyze_threads.php (main entry)
 * - Called by: cli/cron_hybrid.php (scheduled)
 * - Uses: Database.php, calls Claude API
 * - Reads from: emails table (grouped by thread_id)
 * - Writes to: thread_analysis, email_analysis tables
 *
 * @package EmailIntelligence
 */

namespace EmailIntelligence;

use Exception;
use GuzzleHttp\Client as HttpClient;
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Handler\CurlHandler;

class ThreadAnalyzer
{
    private Database $db;
    private HttpClient $http;
    private array $config;
    private $progressCallback = null;

    private array $stats = [
        'threads_analyzed' => 0,
        'threads_failed' => 0,
        'emails_processed' => 0,
        'quotes_extracted' => 0,
        'inconsistencies_found' => 0,
        'tokens_used' => 0,
        'start_time' => null
    ];

    public function __construct(?string $provider = null, ?string $model = null)
    {
        $this->db = Database::getInstance();

        // Determine which AI provider to use
        // Use passed parameter, fall back to constant, then default to 'claude'
        if ($provider === null) {
            $provider = defined('AI_PROVIDER') ? AI_PROVIDER : 'claude';
        }

        if ($provider === 'openai' && defined('OPENAI_API_KEY') && !empty(OPENAI_API_KEY)) {
            // Determine OpenAI model
            // If model is passed, use it; otherwise use OPENAI_MODEL constant or default
            $selectedModel = $model ?? (defined('OPENAI_MODEL') ? OPENAI_MODEL : 'gpt-4o');

            $this->config = [
                'provider' => 'openai',
                'api_key' => OPENAI_API_KEY,
                'model' => $selectedModel,
                'max_tokens' => defined('OPENAI_MAX_TOKENS') ? OPENAI_MAX_TOKENS : 16000,
                'endpoint' => 'https://api.openai.com/v1/chat/completions'
            ];

            // Create clean handler stack to avoid New Relic Guzzle infinite loop
            $stack = HandlerStack::create(new CurlHandler());

            $this->http = new HttpClient([
                'timeout' => 180,
                'handler' => $stack,
                'headers' => [
                    'Content-Type' => 'application/json',
                    'Authorization' => 'Bearer ' . $this->config['api_key']
                ]
            ]);
        } else {
            // Default to Claude
            // Determine Claude model
            // If model is passed, use it; otherwise use CLAUDE_MODEL constant or default
            $selectedModel = $model ?? (defined('CLAUDE_MODEL') ? CLAUDE_MODEL : 'claude-sonnet-4-5-20250929');

            $this->config = [
                'provider' => 'claude',
                'api_key' => defined('CLAUDE_API_KEY') ? CLAUDE_API_KEY : '',
                'model' => $selectedModel,
                'max_tokens' => defined('CLAUDE_MAX_TOKENS') ? CLAUDE_MAX_TOKENS : 16000,
                'endpoint' => 'https://api.anthropic.com/v1/messages'
            ];

            // Create clean handler stack to avoid New Relic Guzzle infinite loop
            $stack = HandlerStack::create(new CurlHandler());

            $this->http = new HttpClient([
                'timeout' => 180,
                'handler' => $stack,
                'headers' => [
                    'Content-Type' => 'application/json',
                    'x-api-key' => $this->config['api_key'],
                    'anthropic-version' => '2023-06-01'
                ]
            ]);
        }
    }

    public function isConfigured(): bool
    {
        return !empty($this->config['api_key']);
    }

    public function getProvider(): string
    {
        return $this->config['provider'] ?? 'claude';
    }

    public function setProgressCallback(callable $callback): void
    {
        $this->progressCallback = $callback;
    }

    private function reportProgress(string $message, ?int $percent = null): void
    {
        $this->logMessage($message);
        if ($this->progressCallback) {
            call_user_func($this->progressCallback, $message, $percent, $this->stats);
        }
    }

    /**
     * Analyze threads that need analysis
     *
     * @param int $batchSize Number of threads to analyze
     * @param int $delayMs Delay between API calls
     * @return array Statistics
     */
    public function analyzeBatch(int $batchSize = 10, int $delayMs = 1000): array
    {
        if (!$this->isConfigured()) {
            throw new Exception("Claude API key not configured");
        }

        $this->stats['start_time'] = microtime(true);
        $this->reportProgress("Starting thread analysis batch", 0);

        // Get threads that need analysis
        $threads = $this->getThreadsNeedingAnalysis($batchSize);

        if (empty($threads)) {
            $this->reportProgress("No threads need analysis", 100);
            return $this->stats;
        }

        $total = count($threads);
        $this->reportProgress("Found {$total} threads to analyze", 5);

        foreach ($threads as $index => $thread) {
            $percent = (int)((($index + 1) / $total) * 100);

            try {
                $this->reportProgress(
                    "Analyzing thread " . ($index + 1) . "/{$total}: " . substr($thread['subject'] ?? 'No subject', 0, 50),
                    $percent
                );

                $this->analyzeThread($thread['thread_id']);
                $this->stats['threads_analyzed']++;

                // Rate limiting
                if ($index < $total - 1 && $delayMs > 0) {
                    usleep($delayMs * 1000);
                }

            } catch (Exception $e) {
                $this->stats['threads_failed']++;
                $this->logMessage("Failed to analyze thread {$thread['thread_id']}: " . $e->getMessage(), 'error');
            }
        }

        $duration = round(microtime(true) - $this->stats['start_time'], 2);
        $this->reportProgress(
            "Batch complete in {$duration}s. Analyzed: {$this->stats['threads_analyzed']}, Failed: {$this->stats['threads_failed']}",
            100
        );

        return $this->stats;
    }

    /**
     * Get threads that need analysis
     *
     * @param int $limit
     * @return array
     */
    private function getThreadsNeedingAnalysis(int $limit): array
    {
        // Get threads that either:
        // 1. Have never been analyzed (not in thread_analysis)
        // 2. Need re-analysis (needs_reanalysis = 1)
        return $this->db->fetchAll(
            "SELECT
                e.thread_id,
                MIN(e.subject) as subject,
                MAX(e.date) as last_email_date,
                COUNT(*) as email_count,
                MAX(e.contact_id) as contact_id
             FROM emails e
             LEFT JOIN thread_analysis ta ON e.thread_id = ta.thread_id
             WHERE e.thread_id IS NOT NULL
               AND e.thread_id != ''
               AND (ta.id IS NULL OR ta.needs_reanalysis = 1)
             GROUP BY e.thread_id
             HAVING email_count >= 1
             ORDER BY last_email_date DESC
             LIMIT ?",
            [$limit]
        );
    }

    /**
     * Analyze a complete email thread
     *
     * @param string $threadId Gmail thread ID
     * @return array Analysis results
     */
    public function analyzeThread(string $threadId): array
    {
        // Get all emails in thread

        if (isset($_REQUEST['threadID'])){
            $extraSQL = " WHERE thread_id = '{$_REQUEST['threadID']}'";
            $emails = $this->db->fetchAll(
                "SELECT * FROM emails
                 {$extraSQL}
                 ORDER BY date ASC"
            );
        }else{
            $emails = $this->db->fetchAll(
                "SELECT * FROM emails
                 WHERE thread_id = ?
                 ORDER BY date ASC",
                [$threadId]
            );
        }
        //echo $threadId;
        //exit();

        if (empty($emails)) {
            throw new Exception("No emails found for thread: {$threadId}");
        }

        $this->stats['emails_processed'] += count($emails);

        // RAG: Retrieve similar past threads for context
        $ragContext = $this->retrieveSimilarThreads($emails);

        // Build thread context for Claude with RAG context
        $prompt = $this->buildThreadAnalysisPrompt($emails, $ragContext);

        //file_put_contents("logs/__ddbgA.log","\n emails: ".print_r($emails, true), FILE_APPEND);
        //file_put_contents("logs/__ddbgA.log","\n argContext: ".print_r($ragContext, true), FILE_APPEND);
        //echo 222;
        //print_r($this->db);
        //exit();
        // Call AI API (Claude or OpenAI)
        $this->logToDb('info', 'ThreadAnalyzer', "Starting analysis for thread {$threadId} with " . count($emails) . " emails", [
            'thread_id' => $threadId,
            'email_count' => count($emails),
            'provider' => $this->config['provider'] ?? 'claude'
        ], $threadId);

        $response = $this->callClaudeAPI($prompt, $threadId);
        //print_r($response);
        //exit();
        file_put_contents("../logs/__ddbgA.log","\n prompt: ".print_r($prompt, true), FILE_APPEND);
        file_put_contents("../logs/__ddbgA.log","\n resp: ".print_r($response, true), FILE_APPEND);
        //exit();
        // Parse response
        $analysis = $this->parseAnalysisResponse($response);

        // Store analysis
        $this->storeThreadAnalysis($threadId, $emails, $analysis, $response);

        // Extract and store quotes
        if (!empty($analysis['quotes'])) {
            $this->stats['quotes_extracted'] += count($analysis['quotes']);
            $this->storeQuotes($emails, $analysis['quotes']);
        }

        // Count inconsistencies
        if (!empty($analysis['inconsistencies'])) {
            $this->stats['inconsistencies_found'] += count($analysis['inconsistencies']);
        }

        // Mark all emails in thread as analyzed
        $this->db->execute(
            "UPDATE emails SET is_analyzed = 1 WHERE thread_id = ?",
            [$threadId]
        );

        return $analysis;
    }

    /**
     * Retrieve similar past threads for RAG context
     *
     * @param array $emails Current thread emails
     * @return array Similar threads with their analyses
     */
    private function retrieveSimilarThreads(array $emails): array
    {
        try {
            // Build search query from thread content
            $searchQuery = $this->buildThreadSearchQuery($emails);

            // Search vector memory for similar threads
            $vectorMemory = new VectorMemory();
            $similarResults = $vectorMemory->search($searchQuery, 3, ['type' => 'thread_summary']);

            $similarThreads = [];
            foreach ($similarResults as $result) {
                // Get thread analysis from database
                $threadId = $result['metadata']['thread_id'] ?? null;
                if ($threadId) {
                    $analysis = $this->db->fetchOne(
                        "SELECT ta.*, c.name as contact_name
                         FROM thread_analysis ta
                         LEFT JOIN contacts c ON ta.contact_id = c.id
                         WHERE ta.thread_id = ?",
                        [$threadId]
                    );

                    if ($analysis) {
                        $similarThreads[] = [
                            'similarity' => $result['similarity'] ?? 0,
                            'thread_id' => $threadId,
                            'contact_name' => $analysis['contact_name'],
                            'summary' => $result['content'] ?? '',
                            'deception_score' => $result['metadata']['deception_score'] ?? null,
                            'trust_score' => $result['metadata']['trust_score'] ?? null,
                            'risk_level' => $result['metadata']['risk_level'] ?? null
                        ];
                    }
                }
            }

            return $similarThreads;

        } catch (Exception $e) {
            error_log("RAG retrieval failed: " . $e->getMessage());
            return []; // Continue without RAG context if it fails
        }
    }

    /**
     * Build search query from thread emails
     *
     * @param array $emails Thread emails
     * @return string Search query
     */
    private function buildThreadSearchQuery(array $emails): string
    {
        // Extract key topics from thread
        $subjects = array_unique(array_column($emails, 'subject'));
        $mainSubject = $subjects[0] ?? 'discussion';

        // Get snippet from first and last email
        $firstBody = strip_tags($emails[0]['body_text'] ?? $emails[0]['body_html'] ?? '');
        $lastBody = strip_tags($emails[count($emails) - 1]['body_text'] ?? '');

        $snippet = substr($firstBody, 0, 200) . ' ... ' . substr($lastBody, 0, 200);

        return "Thread about: {$mainSubject}. {$snippet}";
    }

    /**
     * Build analysis prompt for complete thread
     *
     * @param array $emails All emails in thread
     * @param array $ragContext Similar past threads for context
     * @return string
     */
    private function buildThreadAnalysisPrompt(array $emails, array $ragContext = []): string
    {
        $emailCount = count($emails);
        $firstEmail = $emails[0];
        $lastEmail = $emails[$emailCount - 1];

        // Build conversation timeline
        $timeline = "";
        foreach ($emails as $index => $email) {
            $num = $index + 1;
            $direction = $email['direction'] === 'inbound' ? 'FROM THEM' : 'FROM US';
            $date = date('Y-m-d H:i', strtotime($email['date']));
            $subject = $email['subject'] ?? '(No Subject)';

            // Get email body and strip quoted content
            $body = $email['body_text'] ?? $email['body_html'] ?? '';
            $body = $this->stripQuotedContent($body);

            // Truncate long emails
            if (strlen($body) > 2000) {
                $body = substr($body, 0, 2000) . "\n[... email truncated ...]";
            }

            $timeline .= "━━━ EMAIL #{$num} ({$direction}) ━━━\n";
            $timeline .= "Date: {$date}\n";
            $timeline .= "Subject: {$subject}\n";
            $timeline .= "Content:\n{$body}\n\n";
        }

        // Build RAG context section
        $ragSection = "";
        if (!empty($ragContext)) {
            $ragSection = "\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n";
            $ragSection .= "SIMILAR PAST CONVERSATIONS (for context):\n";
            $ragSection .= "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n";
            $ragSection .= "We found " . count($ragContext) . " similar past situations. Use these as reference:\n\n";

            foreach ($ragContext as $i => $similar) {
                $num = $i + 1;
                $similarity = round($similar['similarity'] * 100, 1);
                $ragSection .= "SIMILAR #{$num} (Similarity: {$similarity}%):\n";
                $ragSection .= "Contact: " . ($similar['contact_name'] ?? 'Unknown') . "\n";
                $ragSection .= "Summary: " . substr($similar['summary'], 0, 300) . "\n";

                if ($similar['deception_score'] !== null) {
                    $ragSection .= "Deception Score: " . round($similar['deception_score'] * 100) . "%\n";
                }
                if ($similar['trust_score'] !== null) {
                    $ragSection .= "Trust Score: " . round($similar['trust_score'] * 100) . "%\n";
                }
                if ($similar['risk_level']) {
                    $ragSection .= "Risk Level: {$similar['risk_level']}\n";
                }
                $ragSection .= "\n";
            }

            $ragSection .= "Use the patterns from these similar situations to inform your analysis.\n";
            $ragSection .= "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n";
        }

        // Build participant list
        $participantStats = [];
        foreach ($emails as $email) {
            $from = $email['from_email'];
            $fromName = $email['from_name'] ?? '';
            if (!isset($participantStats[$from])) {
                $participantStats[$from] = [
                    'name' => $fromName,
                    'email_count' => 0,
                    'direction' => $email['direction'] === 'inbound' ? 'external' : 'self'
                ];
            }
            $participantStats[$from]['email_count']++;
        }

        $participantList = "";
        foreach ($participantStats as $email => $stats) {
            $role = $stats['direction'] === 'self' ? '(US)' : '(THEM)';
            $participantList .= "- {$stats['name']} <{$email}> {$role}: {$stats['email_count']} emails\n";
        }

        // Get owner email from config
        $ownerEmail = defined('GMAIL_USER_EMAIL') ? GMAIL_USER_EMAIL : 'dan@danmarcrm.com';
        $ownerPersona = defined('AI_PERSONA') ? AI_PERSONA : 'Dan runs DanmarCRM for 22 years, developing CRM software and integrations.';

        return <<<PROMPT
You are a behavioral analyst, business strategist, and protective advisor for Dan.

OWNER CONTEXT:
- Owner email: {$ownerEmail}
- {$ownerPersona}
- Dan is the GOOD GUY who must be protected. Identify any threats, manipulation, or harmful patterns from clients.
- Your job: Analyze emails, suggest business opportunities, generate reply options, and protect Dan's interests.

THREAD INFO:
- Total emails: {$emailCount}
- First email: {$firstEmail['date']}
- Last email: {$lastEmail['date']}
- Thread duration: From {$firstEmail['date']} to {$lastEmail['date']}

PARTICIPANTS IN THIS CONVERSATION:
{$participantList}
{$ragSection}
CONVERSATION TIMELINE:
{$timeline}

Analyze this ENTIRE conversation and respond with JSON:

{
  "thread_summary": "<2-3 sentence summary of what this conversation is about and its outcome>",

  "participants_analysis": [
    {
      "email": "<participant's email address>",
      "name": "<participant's name>",
      "role": "<external|self - external=client/supplier, self=us>",
      "email_count": <number of emails they sent>,

      "psychological_profile": {
        "trust_score": <float 0.0-1.0, how trustworthy this person appears>,
        "deception_score": <float 0.0-1.0, likelihood they are being deceptive>,
        "manipulation_score": <float 0.0-1.0, how manipulative their communication is>,
        "reliability_score": <float 0.0-1.0, do they follow through on commitments>,
        "aggression_score": <float 0.0-1.0, level of aggressive/demanding behavior>
      },

      "behavioral_indicators": {
        "communication_style": "<direct|evasive|assertive|passive-aggressive|professional|casual>",
        "consistency": "<consistent|inconsistent|contradictory> - do they change their story?",
        "transparency": "<open|guarded|secretive|deceptive> - how forthcoming are they?",
        "emotional_state": "<calm|anxious|frustrated|angry|desperate|confident>",
        "negotiation_tactics": ["<tactic1>", "<tactic2>"],
        "pressure_tactics": ["<any pressure/manipulation tactics used>"]
      },

      "red_flags": [
        {
          "flag": "<description of concerning behavior>",
          "severity": "<low|medium|high|critical>",
          "evidence": "<quote or reference from their emails>"
        }
      ],

      "positive_signals": [
        {
          "signal": "<positive behavior observed>",
          "evidence": "<quote or reference>"
        }
      ],

      "lies_or_inconsistencies": [
        {
          "claim": "<what they claimed>",
          "contradiction": "<how it contradicts other statements or facts>",
          "emails_involved": "<email numbers where this appears>"
        }
      ],

      "overall_assessment": "<1-2 sentence assessment of this person's trustworthiness and intentions>",
      "risk_level": "<low|medium|high|critical>",
      "verdict": "<trustworthy|cautious|suspicious|untrustworthy>"
    }
  ],

  "sentiment_analysis": {
    "overall_sentiment": "<positive|neutral|negative>",
    "sentiment_score": <float -1.0 to 1.0>,
    "sentiment_trend": "<improving|stable|declining> - how sentiment changed from start to end",
    "tone_progression": "<how the tone evolved through the conversation>"
  },

  "key_points": [
    "<main point 1>",
    "<main point 2>",
    "<main point 3>"
  ],

  "negotiation_analysis": {
    "type": "<price_negotiation|service_request|support|information|complaint|other>",
    "their_stance": "<aggressive|collaborative|passive|demanding|flexible>",
    "our_stance": "<accommodating|firm|flexible|defensive>",
    "outcome": "<resolved|ongoing|abandoned|escalated>",
    "who_initiated": "<them|us>",
    "resolution_summary": "<how it was resolved or current status>"
  },

  "quotes_and_pricing": [
    {
      "email_number": <which email it appeared in>,
      "amount": <number>,
      "currency": "<EUR|USD|etc>",
      "description": "<what it's for>",
      "direction": "<sent_by_us|received_from_them>",
      "type": "<initial|counter|final|discount_request>",
      "context": "<why this quote was given>"
    }
  ],

  "commitments_made": [
    {
      "by": "<them|us>",
      "commitment": "<what was promised>",
      "email_number": <which email>,
      "deadline": "<if mentioned>",
      "was_fulfilled": "<yes|no|unknown|pending> - look if later emails show it was done"
    }
  ],

  "inconsistencies": [
    {
      "type": "<contradiction|changed_story|avoided_question|false_claim>",
      "description": "<what the inconsistency is>",
      "evidence": "<compare email #X where they said A vs email #Y where they said B>",
      "severity": "<low|medium|high>"
    }
  ],

  "deception_indicators": {
    "deception_score": <float 0.0 to 1.0>,
    "trust_score": <float 0.0 to 1.0>,
    "shady_behavior_score": <float 0.0 to 1.0>,
    "red_flags": ["<flag1>", "<flag2>"],
    "hiding_information": "<evidence of withholding relevant information>",
    "manipulation_tactics": ["<any attempts to trick, deceive, or manipulate>"],
    "reasoning": "<why you assigned these scores>"
  },

  "business_risks": {
    "payment_risk": {
      "score": <float 0.0 to 1.0>,
      "payment_delays": "<evidence of delaying or avoiding payments>",
      "excuses_used": ["<list of excuses given for non-payment or delays>"],
      "payment_history_issues": "<any mentioned payment problems>"
    },
    "project_theft_risk": {
      "score": <float 0.0 to 1.0>,
      "knowledge_extraction": "<attempts to extract info to give to competitors>",
      "competitor_mentions": ["<any mentions of other companies doing similar work>"],
      "suspicious_questions": ["<questions that suggest they want to replicate our work elsewhere>"]
    },
    "relationship_risks": [
      {
        "type": "<delay_tactics|scope_creep|blame_shifting|ghosting|false_promises>",
        "description": "<what happened>",
        "evidence": "<email numbers and quotes>"
      }
    ]
  },

  "entities_mentioned": {
    "amounts": [{"value": <num>, "currency": "<curr>", "context": "<what for>"}],
    "dates": [{"date": "<date>", "context": "<what for>"}],
    "products_services": ["<item1>", "<item2>"],
    "companies": ["<company1>"],
    "people": ["<person1>"]
  },

  "action_items": [
    {
      "for": "<them|us>",
      "action": "<what needs to be done>",
      "priority": "<high|medium|low>",
      "deadline": "<if mentioned>"
    }
  ],

  "psychological_insights": {
    "communication_style": "<direct|indirect|formal|casual|aggressive|passive>",
    "decision_making": "<fast|slow|indecisive|confident>",
    "price_sensitivity": "<very_high|high|medium|low> - based on their reactions to quotes",
    "negotiation_skill": "<novice|average|skilled|expert>",
    "reliability_indicators": "<do they follow through? do they change their story?>",
    "pressure_tactics": ["<any tactics they used>"]
  },

  "recommendations": {
    "next_steps": ["<what we should do next>"],
    "how_to_approach": "<advice on how to handle this client/supplier>",
    "pricing_strategy": "<should we quote higher/lower, any insights on their budget>",
    "warnings": ["<things to watch out for with this contact>"],
    "overall_risk_level": "<low|medium|high|critical>",
    "client_verdict": "<trustworthy|cautious|risky|avoid>"
  },

  "reply_suggestions": [
    {
      "label": "<short label like 'Agree & Proceed' or 'Ask Clarification'>",
      "tone": "<positive|neutral|firm|friendly|professional>",
      "preview": "<first 60 chars of reply>",
      "full_text": "<complete ready-to-send email reply with greeting and sign-off>"
    }
  ],

  "business_opportunities": {
    "cross_sell_products": [
      {
        "product_idea": "<what product/service we could offer based on their needs mentioned in emails>",
        "why": "<evidence from emails showing they need this>",
        "how_to_pitch": "<suggested approach to bring this up>"
      }
    ],
    "software_development_ideas": [
      {
        "idea": "<custom software we could build for them based on their problems/needs>",
        "problem_they_have": "<what problem from the emails this solves>",
        "scope_estimate": "<small|medium|large>",
        "pitch_approach": "<how to suggest this to them>"
      }
    ],
    "quote_analysis": {
      "is_quote_related": "<true if this thread is about pricing/quotes, false otherwise>",
      "current_situation": "<summary of quote status from emails>",
      "recommended_action": "<what to propose - accept/counter/hold>",
      "suggested_price_adjustment": "<higher/lower/same and why>",
      "negotiation_leverage": "<what leverage do we have based on emails>",
      "closing_strategy": "<how to close this deal>"
    },
    "persuasion_strategy": {
      "client_motivation": "<what motivates this client based on their emails>",
      "pain_points": ["<pain point 1 from emails>", "<pain point 2>"],
      "questions_to_ask": [
        {
          "question": "<strategic question to ask them>",
          "why_ask": "<what info this reveals>",
          "expected_answers": ["<possible answer 1>", "<possible answer 2>"],
          "follow_up_actions": ["<what to do based on answer 1>", "<what to do based on answer 2>"]
        }
      ],
      "objections_to_expect": ["<likely objection based on emails>"],
      "rebuttals": ["<how to counter each objection>"],
      "buying_signals": ["<positive signals from emails indicating interest>"],
      "recommended_approach": "<overall strategy to win them over>"
    }
  },

  "protective_analysis": {
    "threats_to_owner": [
      {
        "threat_type": "<payment_risk|scope_creep|manipulation|knowledge_theft|reputation_risk|time_waste>",
        "description": "<what the threat is>",
        "severity": "<low|medium|high|critical>",
        "evidence": "<quote from emails showing this threat>",
        "recommended_action": "<how to protect against this>"
      }
    ],
    "harmful_patterns": [
      {
        "pattern": "<pattern name like 'Playing vendors against each other' or 'Endless revision requests'>",
        "evidence": "<proof from emails>",
        "risk_to_dan": "<how this could hurt Dan>"
      }
    ],
    "grammar_analysis": [
      {
        "participant_email": "<email address>",
        "participant_name": "<name>",
        "spelling_errors": "<count of spelling mistakes noticed>",
        "grammar_errors": "<count of grammar issues>",
        "professionalism_score": "<float 0.0-1.0, how professional their writing is>",
        "writing_quality": "<poor|average|good|excellent>",
        "notes": "<observations about their writing style>"
      }
    ],
    "overall_safety_score": "<float 0.0-1.0, how safe is this client for Dan>",
    "protection_alerts": ["<alert 1 if any serious concerns>", "<alert 2>"]
  },

  "churn_risk_signals": {
    "exit_intent_score": <float 0.0-1.0, likelihood they are planning to leave>,
    "exit_signals": [
      {
        "signal": "<what they said or did suggesting they might leave>",
        "severity": "<low|medium|high|critical>",
        "evidence": "<exact quote from email>",
        "email_number": <which email>
      }
    ],
    "competitor_mentions": [
      {
        "competitor": "<name of competitor or alternative mentioned>",
        "context": "<how they mentioned it - comparing, considering, already using?>",
        "threat_level": "<curious|comparing|considering|decided>"
      }
    ],
    "disengagement_signals": {
      "response_slowing": "<true|false - are their responses getting slower?>",
      "enthusiasm_declining": "<true|false - less enthusiastic than before?>",
      "communication_frequency_drop": "<true|false - contacting less often?>",
      "short_replies": "<true|false - replies getting shorter/curt?>"
    },
    "goodbye_language": {
      "detected": <true|false>,
      "examples": ["<farewell phrases used>"],
      "context": "<was it project end, relationship end, or just email sign-off?>"
    },
    "contract_end_signals": {
      "mentioned_ending": <true|false>,
      "mentioned_not_renewing": <true|false>,
      "asked_about_data_export": <true|false>,
      "requested_documentation": <true|false - requesting docs before leaving?>,
      "mentioned_budget_cuts": <true|false>,
      "mentioned_change_in_direction": <true|false>
    },
    "retention_opportunities": ["<what could we do to keep them?>"],
    "churn_risk_summary": "<1-2 sentence summary of churn risk>",
    "recommended_retention_action": "<specific action to prevent churn>"
  },

  "price_perception_analysis": {
    "quote_reaction_type": "<immediate_accept|accept|negotiate|hesitate|reject|shock|ignore|not_applicable>",
    "perceived_value": "<bargain|fair|acceptable|high|very_high|outrageous|unknown>",
    "price_sensitivity_score": <float 0.0-1.0, how sensitive to pricing they are>,
    "willingness_to_pay_more": <float 0.0-1.0, signals they would accept higher prices>,
    "negotiation_aggressiveness": "<passive|mild|moderate|aggressive|very_aggressive>",
    "price_objections": [
      {
        "objection": "<what they said about price>",
        "severity": "<mild|moderate|strong>",
        "email_number": <which email>,
        "response_strategy": "<how to address this objection>"
      }
    ],
    "budget_signals": {
      "budget_mentioned": <true|false>,
      "budget_constraints": "<what constraints they mentioned if any>",
      "budget_flexibility": "<tight|moderate|flexible|generous|unknown>",
      "approval_process_mentioned": <true|false - do they need approval?>
    },
    "value_perception": {
      "understands_value": <true|false>,
      "questioned_value": <true|false>,
      "compared_to_competitors": <true|false>,
      "asked_for_justification": <true|false>
    },
    "discount_behavior": {
      "requested_discount": <true|false>,
      "discount_percent_requested": <number or null>,
      "discount_justification": "<reason they gave for wanting discount>",
      "frequency_of_discount_requests": "<never|rare|occasional|frequent|always>"
    },
    "pricing_strategy_recommendation": "<recommend higher/lower/same pricing and why>",
    "maximum_they_would_pay": "<estimate based on signals, or 'unknown'>"
  },

  "support_satisfaction_analysis": {
    "satisfaction_score": <float 0.0-1.0, overall support satisfaction>,
    "frustration_level": "<none|mild|moderate|high|extreme>",
    "frustration_incidents": [
      {
        "issue": "<what frustrated them>",
        "severity": "<low|medium|high>",
        "our_response": "<how we handled it>",
        "resolution": "<resolved|unresolved|partial>",
        "email_number": <which email>
      }
    ],
    "complaint_patterns": {
      "frequency": "<never|rare|occasional|frequent|constant>",
      "types": ["<type of complaint 1>", "<type 2>"],
      "escalation_tendency": <float 0.0-1.0, likelihood to escalate>
    },
    "patience_level": "<very_patient|patient|neutral|impatient|very_impatient>",
    "appreciation_expressed": <true|false - did they thank us or express appreciation?>,
    "would_recommend_us": "<likely|unlikely|unknown>",
    "relationship_repair_needed": <true|false>,
    "repair_suggestions": ["<how to improve the relationship>"]
  },

  "deep_client_psychology": {
    "decision_maker_analysis": {
      "decision_maker_type": "<primary|influencer|gatekeeper|user|unknown>",
      "authority_level": "<final_authority|needs_approval|recommender|no_authority|unknown>",
      "reports_to_whom": "<if mentioned, who do they need to convince?>",
      "decision_speed": "<impulsive|fast|moderate|slow|very_slow>",
      "decision_criteria": ["<what factors drive their decisions>"]
    },
    "personality_traits": ["<trait1>", "<trait2>", "<trait3>"],
    "communication_preferences": {
      "preferred_channel": "<email|phone|video|in_person|mixed>",
      "response_urgency_expected": "<immediate|same_day|next_day|within_week|flexible>",
      "detail_orientation": "<big_picture|balanced|detail_oriented|extremely_detailed>",
      "emotional_vs_rational": "<very_emotional|emotional|balanced|rational|very_rational>",
      "formality_preference": "<very_casual|casual|neutral|formal|very_formal>"
    },
    "motivators": ["<what motivates them - efficiency, cost savings, innovation, etc>"],
    "fears_and_concerns": ["<what worries them>"],
    "trust_building_opportunities": ["<how to build more trust>"],
    "loyalty_indicators": {
      "loyalty_score": <float 0.0-1.0>,
      "long_term_potential": "<low|medium|high>",
      "referral_likelihood": <float 0.0-1.0>,
      "relationship_investment_level": "<minimal|low|moderate|high|very_high>"
    }
  },

  "software_opportunity_mining": {
    "pain_points_detected": [
      {
        "pain_point": "<specific problem they mentioned>",
        "evidence": "<quote from email>",
        "frequency_mentioned": <how many times mentioned>,
        "urgency": "<low|medium|high|critical>",
        "current_solution": "<how they handle it now, if mentioned>"
      }
    ],
    "automation_opportunities": [
      {
        "process": "<manual process they do that could be automated>",
        "evidence": "<how you know this from emails>",
        "estimated_time_savings": "<low|medium|high>",
        "complexity": "<simple|moderate|complex>",
        "recommended_solution": "<what to build>"
      }
    ],
    "integration_needs": [
      {
        "system_a": "<one system they use>",
        "system_b": "<another system that needs connecting>",
        "evidence": "<how you know from emails>",
        "complexity": "<simple|moderate|complex>"
      }
    ],
    "data_problems": [
      {
        "problem": "<data issue they have - scattered, duplicates, manual entry, etc>",
        "evidence": "<from emails>",
        "solution_type": "<migration|cleanup|integration|dashboard>"
      }
    ],
    "industry_context": {
      "client_industry": "<their industry based on emails>",
      "industry_specific_needs": ["<common needs in this industry>"],
      "regulatory_requirements": ["<if any compliance needs mentioned>"]
    },
    "specific_software_proposals": [
      {
        "title": "<name of software/tool to propose>",
        "problem_solved": "<what it solves for them>",
        "scope": "<tiny|small|medium|large|enterprise>",
        "estimated_hours": "<range like 40-80>",
        "pitch_approach": "<how to bring this up>",
        "win_probability": <float 0.0-1.0>
      }
    ]
  }

}

CRITICAL INSTRUCTIONS:
- BE CONCISE. Keep all text fields SHORT (1-2 sentences max).
- Profile EACH participant separately in participants_analysis
- SKIP per_email_analysis - too verbose
- Only include 3-5 most important red_flags, key_points, action_items
- Only include quotes_and_pricing if actual prices are mentioned
- Look for deception, payment delays, manipulation, knowledge extraction attempts
- Omit empty arrays - if no red_flags found, use empty array []

REPLY SUGGESTIONS (reply_suggestions):
- Generate exactly 5 different reply options based on the conversation
- Each reply must be COMPLETE and ready to copy-paste and send
- Include proper greeting (Hi [name],) and sign-off (Best regards, Dan)
- Options should cover: positive response, clarifying question, boundary-setting, solution-focused, brief acknowledgment
- Base the replies on ACTUAL email content - reference specific things they mentioned

BUSINESS OPPORTUNITIES (business_opportunities):
- Cross-sell: Look at what they're discussing - what OTHER products/services could help them?
- Software ideas: What custom software could solve problems they mentioned in emails?
- Quote analysis: Only fill if this is about pricing/quotes, otherwise set is_quote_related to false
- Persuasion: What questions should Dan ask? What objections might come? How to close?
- ALL insights must come from the EMAIL CONTENT - not generic advice

PROTECTIVE ANALYSIS (protective_analysis):
- Dan (owner) is the GOOD GUY - protect his interests
- Look for: payment risks, scope creep, manipulation, knowledge theft, time wasters
- Grammar analysis: Count actual errors you see in their emails, assess professionalism
- Be SPECIFIC with evidence - quote the emails
- overall_safety_score: 1.0 = completely safe, 0.0 = avoid this client

CHURN RISK SIGNALS (churn_risk_signals) - CRITICAL FOR BUSINESS SURVIVAL:
- Look for ANY hint they might leave: competitor mentions, budget issues, dissatisfaction
- "Goodbye language" = farewell phrases beyond normal email sign-offs
- Disengagement = slower responses, shorter replies, less enthusiasm
- Contract signals = asking about data export, documentation, ending dates
- BE SENSITIVE to subtle signals - even small hints of leaving are valuable
- exit_intent_score: 0.0 = loyal, 1.0 = definitely leaving soon

PRICE PERCEPTION ANALYSIS (price_perception_analysis) - CRITICAL FOR PRICING STRATEGY:
- How did they react to quotes? Shock, acceptance, negotiation?
- Did they ask for discounts? How aggressively?
- Do they understand the value we provide?
- Could we charge MORE? (high willingness_to_pay_more = we're underpricing)
- What is their budget reality? Tight or flexible?
- Maximum they would pay = your best estimate based on all signals

SUPPORT SATISFACTION (support_satisfaction_analysis):
- Any frustration expressed? How severe?
- Did they complain? About what? How often?
- Did they escalate issues?
- Express appreciation or just demands?
- Would they recommend us to others?

DEEP CLIENT PSYCHOLOGY (deep_client_psychology):
- Are they the decision maker or need approval?
- What personality traits are evident?
- What motivates them? Fears?
- How loyal are they? Referral potential?

SOFTWARE OPPORTUNITY MINING (software_opportunity_mining) - REVENUE OPPORTUNITY:
- What PAIN POINTS did they mention? (manual processes, spreadsheet hell, data problems)
- What could be AUTOMATED for them?
- What systems need INTEGRATION?
- What SOFTWARE could Dan build to solve their problems?
- Be SPECIFIC - propose actual software with scope estimates
- This is how Dan finds new work to stay in business!

BE A SKEPTICAL INVESTIGATOR - examine each person's behavior critically.
Look for evidence of deception, manipulation, and hidden agendas in each participant.
Be objective and evidence-based but err on the side of caution.
Respond ONLY with the JSON, no additional text

PROMPT;
    }

    /**
     * Public method to ask Claude a question and get text response
     * Used by API endpoints for ask_ai and generate_reply features
     */
    public function askClaude(string $prompt): string
    {
        $response = $this->callClaudeAPI($prompt);
        return $response['content'] ?? '';
    }

    /**
     * Call AI API (supports both Claude and OpenAI)
     */
    private function callClaudeAPI(string $prompt, ?string $threadId = null): array
    {
        try {
            // Clean UTF-8 in prompt to avoid json_encode errors
            $cleanPrompt = $this->cleanUtf8($prompt);

            $provider = $this->config['provider'] ?? 'claude';
            // Generate unique request ID to track duplicate calls
            $ddbg = debug_backtrace(2);
            $requestId = uniqid('req_', true);
            file_put_contents(__DIR__ . '/../logs/prompt.log', "\n\nDate: " . date('Y-m-d H:i:s') . " [RequestID: {$requestId}] [ThreadID: " . ($threadId ?? 'N/A') . "] - Prompt: " . $cleanPrompt . "\n", FILE_APPEND);
            file_put_contents(__DIR__ . '/../logs/prompt.log', "\n\nDebug Backtrace: " . print_r($ddbg, true) . "\n", FILE_APPEND);
            // Log the request
            $this->logToDb('info', 'AI-Request', "Sending request to {$provider} ({$this->config['model']})", [
                'provider' => $provider,
                'model' => $this->config['model'],
                'max_tokens' => $this->config['max_tokens'],
                'prompt_length' => strlen($cleanPrompt),
                'prompt_preview' => substr($cleanPrompt, 0, 500) . '...',
                'prompt' => $cleanPrompt  // Full prompt sent to AI
            ], $threadId);

            if ($provider === 'openai') {
                // OpenAI request format
                $requestBody = [
                    'model' => $this->config['model'],
                    'max_tokens' => $this->config['max_tokens'],
                    'messages' => [
                        ['role' => 'user', 'content' => $cleanPrompt]
                    ]
                ];
            } else {
                // Claude request format
                $requestBody = [
                    'model' => $this->config['model'],
                    'max_tokens' => $this->config['max_tokens'],
                    'messages' => [
                        ['role' => 'user', 'content' => $cleanPrompt]
                    ]
                ];
            }

            $jsonBody = $this->safeJsonEncode($requestBody);
            $startTime = microtime(true);

            $response = $this->http->post($this->config['endpoint'], [
                'headers' => [
                    'Content-Type' => 'application/json'
                ],
                'body' => $jsonBody
            ]);

            $elapsedTime = round(microtime(true) - $startTime, 2);
            $body = json_decode($response->getBody()->getContents(), true);

            // Parse response based on provider
            if ($provider === 'openai') {
                if (!isset($body['choices'][0]['message']['content'])) {
                    throw new Exception("Invalid OpenAI response structure: " . json_encode($body));
                }
                $content = $body['choices'][0]['message']['content'];
                $usage = $body['usage'] ?? [];
                if (isset($usage['total_tokens'])) {
                    $this->stats['tokens_used'] += $usage['total_tokens'];
                }
            } else {
                if (!isset($body['content'][0]['text'])) {
                    throw new Exception("Invalid Claude response structure: " . json_encode($body));
                }
                $content = $body['content'][0]['text'];
                $usage = $body['usage'] ?? [];
                if (isset($usage)) {
                    $this->stats['tokens_used'] += ($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0);
                }
            }

            // Log the response
            $this->logToDb('info', 'AI-Response', "Received response from {$provider} in {$elapsedTime}s", [
                'provider' => $provider,
                'model' => $body['model'] ?? $this->config['model'],
                'elapsed_seconds' => $elapsedTime,
                'usage' => $usage,
                'response_length' => strlen($content),
                'response_preview' => substr($content, 0, 500) . '...',
                'response' => $content  // Full response from AI
            ], $threadId);

            // Save full request/response to disk for debugging
            $this->saveAnalysisLog($threadId, $cleanPrompt, $content, $usage, $elapsedTime, $provider);

            return [
                'content' => $content,
                'model' => $body['model'] ?? $this->config['model'],
                'usage' => $usage,
                'provider' => $provider
            ];

        } catch (Exception $e) {
            $this->logMessage("API request failed ({$this->config['provider']}): " . $e->getMessage(), 'error');
            $this->logToDb('error', 'AI-Error', "API request failed: " . $e->getMessage(), [
                'provider' => $this->config['provider'] ?? 'unknown',
                'error' => $e->getMessage()
            ], $threadId);
            throw new Exception("{$this->config['provider']} API error: " . $e->getMessage());
        }
    }

    /**
     * Parse JSON response from Claude
     */
    private function parseAnalysisResponse(array $response): array
    {
        $content = $response['content'];

        // Strip markdown code block wrappers if present
        $content = preg_replace('/^```(?:json)?\s*/i', '', $content);
        $content = preg_replace('/\s*```\s*$/', '', $content);
        $content = trim($content);

        // Detect AI refusal responses (content moderation)
        $refusalPatterns = [
            '/I\'m sorry.*can\'t assist/i',
            '/I cannot help with/i',
            '/I\'m not able to/i',
            '/I apologize.*cannot/i',
            '/I\'m unable to/i',
            '/This request.*violates/i',
            '/I can\'t provide/i',
            '/I cannot provide/i'
        ];

        foreach ($refusalPatterns as $pattern) {
            if (preg_match($pattern, $content)) {
                $this->logMessage("AI refused to analyze. Response: " . substr($content, 0, 500), 'warning');
                throw new Exception("AI declined to analyze this thread. This may happen with sensitive content. Try again or contact support.");
            }
        }

        $jsonStart = strpos($content, '{');
        $jsonEnd = strrpos($content, '}');

        if ($jsonStart === false || $jsonEnd === false) {
            $this->logMessage("No JSON found in response. Content: " . substr($content, 0, 500), 'error');
            throw new Exception("No JSON found in API response. The AI may have returned an unexpected response format.");
        }

        $jsonString = substr($content, $jsonStart, $jsonEnd - $jsonStart + 1);

        // Try to fix common JSON issues
        // Remove trailing commas before } or ]
        $jsonString = preg_replace('/,\s*([\}\]])/', '$1', $jsonString);

        $analysis = json_decode($jsonString, true);

        if (json_last_error() !== JSON_ERROR_NONE) {
            // Log the full JSON to file for debugging
            file_put_contents(__DIR__ . '/../logs/failed_json.log', "[" . date('Y-m-d H:i:s') . "]\n" . $jsonString . "\n\n", FILE_APPEND);
            $this->logMessage("JSON parse failed. Error: " . json_last_error_msg() . ". Full JSON logged to failed_json.log", 'error');
            throw new Exception("Failed to parse JSON: " . json_last_error_msg());
        }

        return $analysis;
    }

    /**
     * Store thread analysis results
     */
    private function storeThreadAnalysis(string $threadId, array $emails, array $analysis, array $rawResponse): void
    {
        $firstEmail = $emails[0];
        $lastEmail = $emails[count($emails) - 1];
        $sentiment = $analysis['sentiment_analysis'] ?? [];
        $negotiation = $analysis['negotiation_analysis'] ?? [];
        $deception = $analysis['deception_indicators'] ?? [];

        // Store in thread_analysis table with full analysis
        // Ensure summary is a string (Claude might return it as an array)
        $summary = $analysis['thread_summary'] ?? null;
        if (is_array($summary)) {
            $summary = implode(' ', $summary);
        }

        $this->db->insertOrUpdate('thread_analysis', [
            'thread_id' => $threadId,
            'contact_id' => $lastEmail['contact_id'] ?? null,
            'email_count' => count($emails),
            'last_email_date' => $lastEmail['date'],
            'last_email_id' => $lastEmail['id'],
            'is_analyzed' => 1,
            'needs_reanalysis' => 0,
            'analyzed_at' => date('Y-m-d H:i:s'),
            'summary' => $summary,
            'raw_analysis' => $this->safeJsonEncode($analysis),
            'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
            'deception_score' => $deception['deception_score'] ?? 0
        ], ['contact_id', 'email_count', 'last_email_date', 'last_email_id', 'is_analyzed', 'needs_reanalysis', 'analyzed_at', 'summary', 'raw_analysis', 'key_points', 'deception_score']);

        // Also update MongoDB to keep dashboards in sync
        try {
            $mongo = MongoDatabase::getInstance();
            $mongo->insertOrUpdate('thread_analysis', [
                'thread_id' => $threadId,
                'is_analyzed' => 1,
                'needs_reanalysis' => 0,
                'analyzed_at' => date('Y-m-d H:i:s'),
                'summary' => $summary,
                'raw_analysis' => $this->safeJsonEncode($analysis),
                'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
                'deception_score' => $deception['deception_score'] ?? 0,
                'contact_id' => $lastEmail['contact_id'] ?? null,
                'email_count' => count($emails),
                'last_email_date' => $lastEmail['date'],
                'last_email_id' => $lastEmail['id']
            ], ['thread_id']);
        } catch (Exception $mongoEx) {
            // Log but don't fail the analysis if MongoDB sync fails
            error_log("MongoDB sync failed for thread {$threadId}: " . $mongoEx->getMessage());
        }

        // Store detailed analysis in email_analysis (linked to last email)
        // Ensure string fields are not arrays
        $tone = $negotiation['their_stance'] ?? null;
        if (is_array($tone)) {
            $tone = implode(', ', $tone);
        }
        $sentimentLabel = $sentiment['overall_sentiment'] ?? null;
        if (is_array($sentimentLabel)) {
            $sentimentLabel = is_array($sentimentLabel) ? implode(', ', $sentimentLabel) : $sentimentLabel;
        }

        $this->db->insertOrUpdate('email_analysis', [
            'email_id' => $lastEmail['id'],
            'sentiment_score' => $sentiment['sentiment_score'] ?? null,
            'sentiment_label' => $sentimentLabel,
            'tone' => $tone,
            'urgency_level' => null,
            'key_entities' => isset($analysis['entities_mentioned']) ? $this->safeJsonEncode($analysis['entities_mentioned']) : null,
            'commitments' => isset($analysis['commitments_made']) ? $this->safeJsonEncode($analysis['commitments_made']) : null,
            'requests' => null,
            'action_items' => isset($analysis['action_items']) ? $this->safeJsonEncode($analysis['action_items']) : null,
            'deception_flags' => isset($analysis['inconsistencies']) ? $this->safeJsonEncode($analysis['inconsistencies']) : null,
            'deception_score' => $deception['deception_score'] ?? null,
            'contains_quote' => !empty($analysis['quotes_and_pricing']) ? 1 : 0,
            'quote_amounts' => isset($analysis['quotes_and_pricing']) ? $this->safeJsonEncode($analysis['quotes_and_pricing']) : null,
            'summary' => $summary,
            'key_points' => isset($analysis['key_points']) ? $this->safeJsonEncode($analysis['key_points']) : null,
            'raw_analysis' => $this->safeJsonEncode($analysis),
            'model_used' => $rawResponse['model'] ?? $this->config['model'],
            'tokens_used' => ($rawResponse['usage']['input_tokens'] ?? 0) + ($rawResponse['usage']['output_tokens'] ?? 0),
            'analyzed_at' => date('Y-m-d H:i:s')
        ], ['email_id']);

        // Store all participants with their analysis (non-fatal - don't let this kill the response)
        try {
            $this->storeThreadParticipants($threadId, $analysis);
        } catch (\Throwable $e) {
            $this->logMessage("storeThreadParticipants failed (non-fatal): " . $e->getMessage(), 'warning');
        }

        // Log success to DB
        $this->logToDb('info', 'Thread analysis completed', $threadId, [
            'email_count' => count($emails),
            'participants' => count($analysis['participants_analysis'] ?? []),
            'deception_score' => $deception['deception_score'] ?? 0
        ]);
    }

    /**
     * Store extracted quotes
     */
    private function storeQuotes(array $emails, array $quotes): void
    {
        foreach ($quotes as $quote) {
            if (empty($quote['amount'])) continue;

            // Find which email in thread this quote came from
            $emailIndex = ($quote['email_number'] ?? 1) - 1;
            $email = $emails[$emailIndex] ?? $emails[count($emails) - 1];

            try {
                $this->db->insert('quotes', [
                    'contact_id' => $email['contact_id'],
                    'email_id' => $email['id'],
                    'amount' => $quote['amount'],
                    'currency' => $quote['currency'] ?? 'EUR',
                    'description' => $quote['description'] ?? $quote['context'] ?? null,
                    'direction' => $quote['direction'] === 'sent_by_us' ? 'sent' : 'received',
                    'quote_type' => $quote['type'] ?? 'initial',
                    'outcome' => 'pending',
                    'confidence_score' => 0.8,
                    'quote_date' => $email['date']
                ]);
            } catch (Exception $e) {
                $this->logMessage("Failed to store quote: " . $e->getMessage(), 'warning');
            }
        }
    }

    /**
     * Store thread participants (all contacts in the thread with their analysis)
     */
    private function storeThreadParticipants(string $threadId, array $analysis): void
    {
        $participants = $analysis['participants_analysis'] ?? [];

        foreach ($participants as $participant) {
            $email = $participant['email'] ?? null;
            if (!$email) continue;

            // Find or create contact
            $contact = $this->db->fetchOne("SELECT id FROM contacts WHERE email = ?", [$email]);
            if (!$contact) {
                // Create new contact
                $this->db->insert('contacts', [
                    'email' => $email,
                    'name' => $participant['name'] ?? null,
                    'type' => $participant['role'] === 'self' ? 'internal' : 'client',
                    'created_at' => date('Y-m-d H:i:s')
                ]);
                // lastInsertId() requires PDO (MySQL). Fall back to SELECT when using MongoDB.
                try {
                    $contactId = $this->db->lastInsertId();
                } catch (\Throwable $e) {
                    $newContact = $this->db->fetchOne("SELECT id FROM contacts WHERE email = ?", [$email]);
                    $contactId = $newContact['id'] ?? null;
                }
            } else {
                $contactId = $contact['id'];
            }
            if (!$contactId) continue; // Skip if we couldn't get a contact ID

            $profile = $participant['psychological_profile'] ?? [];
            $behavioral = $participant['behavioral_indicators'] ?? [];

            // Store participant in thread_participants table
            try {
                $this->db->insertOrUpdate('thread_participants', [
                    'thread_id' => $threadId,
                    'contact_id' => $contactId,
                    'email_count' => $participant['email_count'] ?? 0,
                    'role' => $participant['role'] === 'self' ? 'self' : 'external',
                    'trust_score' => $profile['trust_score'] ?? null,
                    'deception_score' => $profile['deception_score'] ?? null,
                    'manipulation_score' => $profile['manipulation_score'] ?? null,
                    'reliability_score' => $profile['reliability_score'] ?? null,
                    'risk_level' => $participant['risk_level'] ?? null,
                    'verdict' => $participant['verdict'] ?? null,
                    'psychological_profile' => $this->safeJsonEncode($profile),
                    'behavioral_indicators' => $this->safeJsonEncode($behavioral),
                    'red_flags' => $this->safeJsonEncode($participant['red_flags'] ?? []),
                    'positive_signals' => $this->safeJsonEncode($participant['positive_signals'] ?? []),
                    'overall_assessment' => $participant['overall_assessment'] ?? null
                ], ['email_count', 'role', 'trust_score', 'deception_score', 'manipulation_score',
                    'reliability_score', 'risk_level', 'verdict', 'psychological_profile',
                    'behavioral_indicators', 'red_flags', 'positive_signals', 'overall_assessment']);

                // Store learned behaviors from this participant
                $this->storeLearnedBehaviors($contactId, $threadId, $participant);

            } catch (Exception $e) {
                $this->logMessage("Failed to store participant {$email}: " . $e->getMessage(), 'warning');
            }
        }
    }

    /**
     * Store learned behaviors for RAG context in future analyses
     */
    private function storeLearnedBehaviors(int $contactId, string $threadId, array $participant): void
    {
        // Store red flags as learned behaviors
        $redFlags = $participant['red_flags'] ?? [];
        foreach ($redFlags as $flag) {
            $this->addLearnedBehavior($contactId, $threadId, 'red_flag', $flag['flag'] ?? '', $flag['evidence'] ?? '', $flag['severity'] ?? 'medium');
        }

        // Store positive signals
        $positives = $participant['positive_signals'] ?? [];
        foreach ($positives as $signal) {
            $this->addLearnedBehavior($contactId, $threadId, 'positive_signal', $signal['signal'] ?? '', $signal['evidence'] ?? '', 'low');
        }

        // Store negotiation tactics
        $tactics = $participant['behavioral_indicators']['negotiation_tactics'] ?? [];
        foreach ($tactics as $tactic) {
            if (is_string($tactic)) {
                $this->addLearnedBehavior($contactId, $threadId, 'negotiation_tactic', $tactic, '', 'medium');
            }
        }

        // Store pressure tactics as potential deception indicators
        $pressure = $participant['behavioral_indicators']['pressure_tactics'] ?? [];
        foreach ($pressure as $tactic) {
            if (is_string($tactic)) {
                $this->addLearnedBehavior($contactId, $threadId, 'deception_indicator', $tactic, '', 'medium');
            }
        }

        // Store communication pattern
        $style = $participant['behavioral_indicators']['communication_style'] ?? null;
        if ($style) {
            $this->addLearnedBehavior($contactId, $threadId, 'communication_pattern', $style, '', 'low');
        }
    }

    /**
     * Add or update a learned behavior
     */
    private function addLearnedBehavior(int $contactId, string $threadId, string $type, string $description, string $evidence, string $severity): void
    {
        if (empty($description)) return;

        // Check if similar behavior already exists
        $existing = $this->db->fetchOne(
            "SELECT id, frequency FROM learned_behaviors WHERE contact_id = ? AND behavior_type = ? AND description = ?",
            [$contactId, $type, $description]
        );

        if ($existing) {
            // Update frequency and last seen
            $this->db->execute(
                "UPDATE learned_behaviors SET frequency = frequency + 1, last_seen_thread = ?, updated_at = NOW() WHERE id = ?",
                [$threadId, $existing['id']]
            );
        } else {
            // Insert new behavior
            try {
                $this->db->insert('learned_behaviors', [
                    'contact_id' => $contactId,
                    'behavior_type' => $type,
                    'description' => $description,
                    'evidence' => $evidence,
                    'severity' => $severity,
                    'frequency' => 1,
                    'first_seen_thread' => $threadId,
                    'last_seen_thread' => $threadId,
                    'confidence_score' => 0.8,
                    'is_active' => 1
                ]);
            } catch (Exception $e) {
                // Ignore duplicate errors
            }
        }
    }

    /**
     * Log to api_logs table for extension visibility
     */
    public function logToDb(string $level, string $source, string $message, array $context = [], ?string $threadId = null): void
    {
        try {
            $this->db->insert('api_logs', [
                'log_level' => $level,
                'source' => $source,
                'message' => $message,
                'context' => !empty($context) ? $this->safeJsonEncode($context) : null,
                'thread_id' => $threadId,
                'created_at' => date('Y-m-d H:i:s')
            ]);
        } catch (Exception $e) {
            // Fallback to file log
            $this->logMessage("DB log failed: " . $e->getMessage(), 'warning');
        }
    }

    /**
     * Mark thread for re-analysis (when new emails arrive)
     *
     * @param string $threadId
     */
    public function markThreadForReanalysis(string $threadId): void
    {
        $this->db->execute(
            "UPDATE thread_analysis SET needs_reanalysis = 1 WHERE thread_id = ?",
            [$threadId]
        );
    }

    /**
     * Get statistics
     */
    public function getStats(): array
    {
        return $this->stats;
    }

    private function logMessage(string $message, string $level = 'info'): void
    {
        $logFile = defined('LOG_PATH') ? LOG_PATH . 'thread_analyzer.log' : __DIR__ . '/../logs/thread_analyzer.log';
        $timestamp = date('Y-m-d H:i:s');
        $levelUpper = strtoupper($level);
        file_put_contents($logFile, "[{$timestamp}] [{$levelUpper}] {$message}\n", FILE_APPEND);

        if (php_sapi_name() === 'cli') {
            echo "[{$levelUpper}] {$message}\n";
        }
    }

    /**
     * Safely encode data to JSON, fixing UTF-8 issues
     */
    private function safeJsonEncode($data): string
    {
        $cleaned = $this->cleanUtf8($data);
        $json = json_encode($cleaned, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE);
        if ($json === false) {
            $this->logMessage("json_encode error: " . json_last_error_msg(), 'error');
            return '{}';
        }
        return $json;
    }

    /**
     * Recursively clean UTF-8 strings in data
     */
    private function cleanUtf8($data)
    {
        if (is_string($data)) {
            // Remove invalid UTF-8 sequences
            $data = mb_convert_encoding($data, 'UTF-8', 'UTF-8');
            // Remove null bytes and other problematic characters
            $data = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $data);
            return $data;
        }
        if (is_array($data)) {
            return array_map([$this, 'cleanUtf8'], $data);
        }
        return $data;
    }

    /**
     * Save full AI request/response to disk for debugging and review
     * Files are saved in logs/ai_analysis/ organized by date
     */
    private function saveAnalysisLog(?string $threadId, string $prompt, string $response, array $usage, float $elapsedTime, string $provider): void
    {
        try {
            $logDir = dirname(__DIR__) . '/logs/ai_analysis/' . date('Y-m-d');
            if (!is_dir($logDir)) {
                mkdir($logDir, 0755, true);
            }

            $timestamp = date('H-i-s');
            $safeThreadId = preg_replace('/[^a-zA-Z0-9_-]/', '_', $threadId ?? 'unknown');
            $filename = "{$logDir}/{$timestamp}_{$safeThreadId}.txt";

            $tokenInfo = "Input tokens: " . ($usage['input_tokens'] ?? 'N/A') .
                        ", Output tokens: " . ($usage['output_tokens'] ?? 'N/A') .
                        ", Total: " . (($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0));

            $logContent = "=" . str_repeat("=", 79) . "\n";
            $logContent .= "THREAD ANALYSIS LOG\n";
            $logContent .= "=" . str_repeat("=", 79) . "\n\n";
            $logContent .= "Thread ID: {$threadId}\n";
            $logContent .= "Timestamp: " . date('Y-m-d H:i:s') . "\n";
            $logContent .= "Provider: {$provider}\n";
            $logContent .= "Elapsed Time: {$elapsedTime}s\n";
            $logContent .= "Token Usage: {$tokenInfo}\n";
            $logContent .= "Prompt Length: " . strlen($prompt) . " chars\n";
            $logContent .= "Response Length: " . strlen($response) . " chars\n";
            $logContent .= "\n" . str_repeat("-", 80) . "\n";
            $logContent .= "PROMPT SENT TO AI:\n";
            $logContent .= str_repeat("-", 80) . "\n\n";
            $logContent .= $prompt;
            $logContent .= "\n\n" . str_repeat("-", 80) . "\n";
            $logContent .= "AI RESPONSE:\n";
            $logContent .= str_repeat("-", 80) . "\n\n";
            $logContent .= $response;
            $logContent .= "\n\n" . str_repeat("=", 80) . "\n";
            $logContent .= "END OF LOG\n";
            $logContent .= str_repeat("=", 80) . "\n";

            file_put_contents($filename, $logContent);

            // Also keep a summary log for quick reference
            $summaryFile = dirname(__DIR__) . '/logs/ai_analysis/summary.log';
            $summaryLine = date('Y-m-d H:i:s') . " | Thread: {$threadId} | Provider: {$provider} | Time: {$elapsedTime}s | Tokens: " . (($usage['input_tokens'] ?? 0) + ($usage['output_tokens'] ?? 0)) . " | File: {$filename}\n";
            file_put_contents($summaryFile, $summaryLine, FILE_APPEND);

        } catch (Exception $e) {
            // Don't fail the analysis if logging fails
            error_log("Failed to save analysis log: " . $e->getMessage());
        }
    }

    /**
     * Strip quoted content from email body to avoid duplication
     * Removes:
     * - Lines starting with > (quoted replies)
     * - "On X date, Y wrote:" patterns and everything after
     * - "From: ... Sent: ... To: ... Subject: ..." forwarded headers
     */
    private function stripQuotedContent(string $body): string
    {
        // Remove lines starting with > (quoted text)
        $body = preg_replace('/^>+.*$/m', '', $body);

        // Remove "On [date], [person] wrote:" and everything after
        $patterns = [
            '/On\s+.{10,60}\s+wrote:\s*$/mi',  // "On Monday, Jan 1, John wrote:"
            '/On\s+\d{1,2}[\/-]\d{1,2}[\/-]\d{2,4}.+wrote:\s*$/mi',  // "On 01/02/2023, John wrote:"
            '/-{2,}.*Original Message.*-{2,}/i',  // "-----Original Message-----"
            '/From:\s+.+\nSent:\s+.+\nTo:\s+.+\nSubject:\s+.+/i',  // Outlook forwarded header
            '/_{5,}.*$/m',  // _____ separator lines
        ];

        foreach ($patterns as $pattern) {
            if (preg_match($pattern, $body, $matches, PREG_OFFSET_CAPTURE)) {
                // Cut everything from the match onwards
                $body = substr($body, 0, $matches[0][1]);
            }
        }

        // Clean up multiple consecutive newlines
        $body = preg_replace('/\n{3,}/', "\n\n", $body);

        return trim($body);
    }
}

-    private function stripQuotedContent(string $body): string

+    public static function stripQuotedContent(string $body): string


Now let me check if `stripQuotedContent` is called internally with `$this->`:


404:            $body = $this->stripQuotedContent($body);
1616:    public static function stripQuotedContent(string $body): string


Need to update the internal call to use `self::` instead of `$this->`:


   400→            $subject = $email['subject'] ?? '(No Subject)';
   401→
   402→            // Get email body and strip quoted content
   403→            $body = $email['body_text'] ?? $email['body_html'] ?? '';
   404→            $body = $this->stripQuotedContent($body);
   405→
   406→            // Truncate long emails
   407→            if (strlen($body) > 2000) {
   408→                $body = substr($body, 0, 2000) . "\n[... email truncated ...]";
   409→            }

            $subject = $email['subject'] ?? '(No Subject)';

            // Get email body and strip quoted content
            $body = $email['body_text'] ?? $email['body_html'] ?? '';
            $body = $this->stripQuotedContent($body);

            // Truncate long emails
            if (strlen($body) > 2000) {
                $body = substr($body, 0, 2000) . "\n[... email truncated ...]";
            }