#!/usr/bin/env python3
"""
XGBoost深度优化策略
"""

import pandas as pd
import numpy as np
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from sklearn.metrics import accuracy_score, f1_score, classification_report
from sklearn.feature_selection import RFE, SelectFromModel
from sklearn.ensemble import RandomForestClassifier
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from imblearn.combine import SMOTEENN
import matplotlib.pyplot as plt
import seaborn as sns

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False

def load_and_prepare_data():
    """加载和准备数据"""
    from enhanced_sentiment_analyzer import EnhancedAirlineSentimentAnalyzer
    
    df = pd.read_csv('../data/Tweets.csv')
    analyzer = EnhancedAirlineSentimentAnalyzer(random_state=42)
    features, y = analyzer.prepare_features(df)
    
    return features, y, df

def advanced_hyperparameter_tuning(features, y):
    """高级超参数调优"""
    print("\n" + "="*70)
    print("高级超参数调优")
    print("="*70)
    
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    # 策略1：网格搜索优化
    print("\n1. 网格搜索优化")
    
    # 基于消融实验的最佳参数范围
    param_grid = {
        'learning_rate': [0.01, 0.05, 0.1],
        'n_estimators': [100, 200, 300],
        'max_depth': [3, 5, 7],
        'min_child_weight': [1, 3, 5],
        'subsample': [0.7, 0.8, 0.9],
        'colsample_bytree': [0.7, 0.8, 0.9],
        'reg_alpha': [0, 0.1, 0.5],
        'reg_lambda': [0.5, 1.0, 2.0]
    }
    
    # 手动网格搜索（避免中文路径问题）
    best_score = 0
    best_params = {}
    
    # 简化搜索（实际应用中可以使用更复杂的搜索策略）
    test_params = [
        {'learning_rate': 0.05, 'n_estimators': 200, 'max_depth': 3, 'min_child_weight': 3},
        {'learning_rate': 0.1, 'n_estimators': 300, 'max_depth': 5, 'min_child_weight': 1},
        {'learning_rate': 0.05, 'n_estimators': 100, 'max_depth': 3, 'min_child_weight': 5},
    ]
    
    for i, params in enumerate(test_params):
        xgb = XGBClassifier(
            random_state=42,
            eval_metric='mlogloss',
            use_label_encoder=False,
            **params
        )
        
        # 使用交叉验证评估
        cv_scores = cross_val_score(xgb, features, y, cv=5, scoring='f1_macro')
        mean_score = cv_scores.mean()
        
        print(f"   参数组合{i+1}: Macro-F1={mean_score:.4f}")
        
        if mean_score > best_score:
            best_score = mean_score
            best_params = params
    
    print(f"   最佳参数: {best_params}")
    print(f"   最佳Macro-F1: {best_score:.4f}")
    
    return best_params

def handle_class_imbalance(features, y):
    """处理类别不平衡"""
    print("\n" + "="*70)
    print("类别不平衡处理")
    print("="*70)
    
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    strategies = [
        ("原始数据", X_train, y_train),
        ("SMOTE过采样", None, None),
        ("随机欠采样", None, None),
        ("SMOTEENN组合", None, None)
    ]
    
    # 应用采样策略
    smote = SMOTE(random_state=42)
    under = RandomUnderSampler(random_state=42)
    smote_enn = SMOTEENN(random_state=42)
    
    X_smote, y_smote = smote.fit_resample(X_train, y_train)
    X_under, y_under = under.fit_resample(X_train, y_train)
    X_smote_enn, y_smote_enn = smote_enn.fit_resample(X_train, y_train)
    
    strategies[1] = ("SMOTE过采样", X_smote, y_smote)
    strategies[2] = ("随机欠采样", X_under, y_under)
    strategies[3] = ("SMOTEENN组合", X_smote_enn, y_smote_enn)
    
    results = []
    
    for name, X_train_bal, y_train_bal in strategies:
        if X_train_bal is not None:
            xgb = XGBClassifier(
                random_state=42,
                eval_metric='mlogloss',
                use_label_encoder=False,
                learning_rate=0.05,
                n_estimators=200,
                max_depth=3
            )
            
            xgb.fit(X_train_bal, y_train_bal)
            y_pred = xgb.predict(X_test)
            
            acc = accuracy_score(y_test, y_pred)
            f1 = f1_score(y_test, y_pred, average='macro')
            
            results.append((name, acc, f1))
            print(f"   {name}: 准确率={acc:.4f}, Macro-F1={f1:.4f}")
    
    return results

def feature_engineering_enhancement(features, y, df):
    """特征工程增强"""
    print("\n" + "="*70)
    print("特征工程增强")
    print("="*70)
    
    # 策略1：递归特征消除
    print("\n1. 递归特征消除(RFE)")
    
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    # 使用随机森林进行特征选择
    rf = RandomForestClassifier(random_state=42)
    rfe = RFE(estimator=rf, n_features_to_select=15)
    
    X_train_rfe = rfe.fit_transform(X_train, y_train)
    X_test_rfe = rfe.transform(X_test)
    
    xgb_rfe = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False
    )
    
    xgb_rfe.fit(X_train_rfe, y_train)
    y_pred_rfe = xgb_rfe.predict(X_test_rfe)
    
    acc_rfe = accuracy_score(y_test, y_pred_rfe)
    f1_rfe = f1_score(y_test, y_pred_rfe, average='macro')
    
    print(f"   RFE特征选择: 准确率={acc_rfe:.4f}, Macro-F1={f1_rfe:.4f}")
    
    # 策略2：基于重要性的特征选择
    print("\n2. 基于重要性的特征选择")
    
    xgb_imp = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False
    )
    
    xgb_imp.fit(X_train, y_train)
    
    # 选择重要性大于平均值的特征
    importances = xgb_imp.feature_importances_
    threshold = np.mean(importances)
    
    selector = SelectFromModel(xgb_imp, threshold=threshold, prefit=True)
    X_train_imp = selector.transform(X_train)
    X_test_imp = selector.transform(X_test)
    
    xgb_imp2 = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False
    )
    
    xgb_imp2.fit(X_train_imp, y_train)
    y_pred_imp = xgb_imp2.predict(X_test_imp)
    
    acc_imp = accuracy_score(y_test, y_pred_imp)
    f1_imp = f1_score(y_test, y_pred_imp, average='macro')
    
    print(f"   重要性选择: 准确率={acc_imp:.4f}, Macro-F1={f1_imp:.4f}")
    print(f"   选择特征数: {X_train_imp.shape[1]}/{features.shape[1]}")
    
    return [(acc_rfe, f1_rfe), (acc_imp, f1_imp)]

def ensemble_learning(features, y):
    """集成学习策略"""
    print("\n" + "="*70)
    print("集成学习策略")
    print("="*70)
    
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    # 策略1：Bagging集成
    print("\n1. Bagging集成")
    
    from sklearn.ensemble import BaggingClassifier
    
    xgb_base = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False,
        learning_rate=0.05,
        n_estimators=100
    )
    
    bagging = BaggingClassifier(
        estimator=xgb_base,
        n_estimators=10,
        random_state=42
    )
    
    bagging.fit(X_train, y_train)
    y_pred_bag = bagging.predict(X_test)
    
    acc_bag = accuracy_score(y_test, y_pred_bag)
    f1_bag = f1_score(y_test, y_pred_bag, average='macro')
    
    print(f"   Bagging集成: 准确率={acc_bag:.4f}, Macro-F1={f1_bag:.4f}")
    
    # 策略2：Stacking集成
    print("\n2. Stacking集成")
    
    from sklearn.ensemble import StackingClassifier
    from sklearn.linear_model import LogisticRegression
    
    estimators = [
        ('xgb1', XGBClassifier(random_state=42, eval_metric='mlogloss', use_label_encoder=False)),
        ('xgb2', XGBClassifier(random_state=43, eval_metric='mlogloss', use_label_encoder=False, learning_rate=0.1)),
        ('rf', RandomForestClassifier(random_state=42))
    ]
    
    stacking = StackingClassifier(
        estimators=estimators,
        final_estimator=LogisticRegression(random_state=42),
        cv=5
    )
    
    stacking.fit(X_train, y_train)
    y_pred_stack = stacking.predict(X_test)
    
    acc_stack = accuracy_score(y_test, y_pred_stack)
    f1_stack = f1_score(y_test, y_pred_stack, average='macro')
    
    print(f"   Stacking集成: 准确率={acc_stack:.4f}, Macro-F1={f1_stack:.4f}")
    
    return [(acc_bag, f1_bag), (acc_stack, f1_stack)]

def custom_loss_optimization(features, y):
    """自定义损失函数优化"""
    print("\n" + "="*70)
    print("自定义损失函数优化")
    print("="*70)
    
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    # 策略1：类别权重调整
    print("\n1. 类别权重调整")
    
    # 计算类别权重
    class_counts = np.bincount(y)
    total_samples = len(y)
    class_weights = total_samples / (len(class_counts) * class_counts)
    
    # 转换为字典格式
    weight_dict = {i: weight for i, weight in enumerate(class_weights)}
    
    xgb_weighted = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False,
        scale_pos_weight=weight_dict[2] if 2 in weight_dict else 1.0  # positive类别权重
    )
    
    xgb_weighted.fit(X_train, y_train)
    y_pred_weighted = xgb_weighted.predict(X_test)
    
    acc_weighted = accuracy_score(y_test, y_pred_weighted)
    f1_weighted = f1_score(y_test, y_pred_weighted, average='macro')
    
    print(f"   类别权重调整: 准确率={acc_weighted:.4f}, Macro-F1={f1_weighted:.4f}")
    
    # 策略2：F1分数优化
    print("\n2. F1分数优化")
    
    # 使用概率阈值调整来优化F1
    xgb_prob = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False
    )
    
    xgb_prob.fit(X_train, y_train)
    y_proba = xgb_prob.predict_proba(X_test)
    
    # 尝试不同的概率阈值
    best_f1 = 0
    best_threshold = 0.5
    
    for threshold in [0.3, 0.4, 0.5, 0.6, 0.7]:
        y_pred_thresh = np.argmax(y_proba, axis=1)
        # 对positive类别应用阈值
        positive_mask = y_proba[:, 2] > threshold
        y_pred_thresh[positive_mask] = 2  # positive类别
        
        f1_thresh = f1_score(y_test, y_pred_thresh, average='macro')
        
        if f1_thresh > best_f1:
            best_f1 = f1_thresh
            best_threshold = threshold
    
    print(f"   最佳阈值: {best_threshold}, Macro-F1={best_f1:.4f}")
    
    return [(acc_weighted, f1_weighted), (None, best_f1)]

def comprehensive_optimization():
    """综合优化策略"""
    print("\n" + "="*70)
    print("XGBoost深度优化综合策略")
    print("="*70)
    
    # 加载数据
    features, y, df = load_and_prepare_data()
    
    # 1. 高级超参数调优
    best_params = advanced_hyperparameter_tuning(features, y)
    
    # 2. 类别不平衡处理
    imbalance_results = handle_class_imbalance(features, y)
    
    # 3. 特征工程增强
    feature_results = feature_engineering_enhancement(features, y, df)
    
    # 4. 集成学习
    ensemble_results = ensemble_learning(features, y)
    
    # 5. 自定义损失函数优化
    loss_results = custom_loss_optimization(features, y)
    
    # 综合评估
    print("\n" + "="*70)
    print("综合优化结果对比")
    print("="*70)
    
    # 收集所有结果
    all_results = []
    
    # 基础模型
    X_train, X_test, y_train, y_test = train_test_split(
        features, y, test_size=0.2, random_state=42, stratify=y
    )
    
    xgb_base = XGBClassifier(
        random_state=42,
        eval_metric='mlogloss',
        use_label_encoder=False
    )
    
    xgb_base.fit(X_train, y_train)
    y_pred_base = xgb_base.predict(X_test)
    
    acc_base = accuracy_score(y_test, y_pred_base)
    f1_base = f1_score(y_test, y_pred_base, average='macro')
    
    all_results.append(("基础模型", acc_base, f1_base))
    
    # 添加其他策略结果
    for name, acc, f1 in imbalance_results:
        all_results.append((name, acc, f1))
    
    all_results.append(("RFE特征选择", feature_results[0][0], feature_results[0][1]))
    all_results.append(("重要性选择", feature_results[1][0], feature_results[1][1]))
    all_results.append(("Bagging集成", ensemble_results[0][0], ensemble_results[0][1]))
    all_results.append(("Stacking集成", ensemble_results[1][0], ensemble_results[1][1]))
    all_results.append(("类别权重", loss_results[0][0], loss_results[0][1]))
    all_results.append(("F1阈值优化", None, loss_results[1][1]))
    
    # 显示结果
    for name, acc, f1 in all_results:
        if acc is not None:
            status = "✅" if acc >= 0.85 or f1 >= 0.80 else "❌"
            print(f"{status} {name:15} 准确率={acc:.4f}, Macro-F1={f1:.4f}")
        else:
            status = "✅" if f1 >= 0.80 else "❌"
            print(f"{status} {name:15} Macro-F1={f1:.4f}")
    
    # 找出最佳策略
    valid_results = [r for r in all_results if r[1] is not None]
    best_strategy = max(valid_results, key=lambda x: x[2])  # 按Macro-F1排序
    
    print(f"\n🏆 最佳策略: {best_strategy[0]}")
    print(f"   准确率: {best_strategy[1]:.4f}")
    print(f"   Macro-F1: {best_strategy[2]:.4f}")
    
    if best_strategy[1] >= 0.85 or best_strategy[2] >= 0.80:
        print("✅ 达到目标性能!")
    else:
        print(f"❌ 距离目标还差: {max(0.85 - best_strategy[1], 0.80 - best_strategy[2]):.4f}")
    
    return best_strategy

if __name__ == "__main__":
    best_result = comprehensive_optimization()
    
    print("\n" + "="*70)
    print("XGBoost深度优化完成")
    print("="*70)
    print(f"最佳策略: {best_result[0]}")
    print(f"最终性能: 准确率={best_result[1]:.4f}, Macro-F1={best_result[2]:.4f}")