import pandas as pd

# Load the dataset
titanic_data = pd.read_csv('data/titanic.csv')
# Display the basic information and the first few rows of the dataset
titanic_data.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 891 entries, 0 to 890
Data columns (total 12 columns):
 #   Column       Non-Null Count  Dtype  
---  ------       --------------  -----  
 0   PassengerId  891 non-null    int64  
 1   Survived     891 non-null    int64  
 2   Pclass       891 non-null    int64  
 3   Name         891 non-null    object 
 4   Sex          891 non-null    object 
 5   Age          714 non-null    float64
 6   SibSp        891 non-null    int64  
 7   Parch        891 non-null    int64  
 8   Ticket       891 non-null    object 
 9   Fare         891 non-null    float64
 10  Cabin        204 non-null    object 
 11  Embarked     889 non-null    object 
dtypes: float64(2), int64(5), object(5)
memory usage: 83.7+ KB
titanic_data.head()

# Inspecting missing values
missing_values = titanic_data.isnull().sum()
missing_values
PassengerId      0
Survived         0
Pclass           0
Name             0
Sex              0
Age            177
SibSp            0
Parch            0
Ticket           0
Fare             0
Cabin          687
Embarked         2
dtype: int64

# Checking data types
data_types = titanic_data.dtypes
data_types
PassengerId      int64
Survived         int64
Pclass           int64
Name            object
Sex             object
Age            float64
SibSp            int64
Parch            int64
Ticket          object
Fare           float64
Cabin           object
Embarked        object
dtype: object

# Identifying potential outliers by looking at summary statistics
summary_statistics = titanic_data.describe()
summary_statistics

# Visual inspectation of popential outliers
import matplotlib.pyplot as plt
import seaborn as sns


# Setting the style for the plots
sns.set(style='whitegrid')
# 使用Seaborn绘制描述统计信息
plt.figure(figsize=(10, 6))
sns.barplot(data=summary_statistics, orient='h')
plt.title('Summary Statistics of Titanic Data')
plt.show()

# Create box plots for 'Age' and 'Fare'
fig, axes = plt.subplots(1, 2, figsize=(15, 5))
sns.boxplot(ax=axes[0], x=titanic_data['Age']).set_title('Box Plot of Age')
sns.boxenplot(ax=axes[1], x=titanic_data['Fare']).set_title('Box Plot of Fare')
plt.show()

# Addressing missing values
# 1. Imputing missing values in 'Age' with the median
median_age = titanic_data['Age'].median()
titanic_data['Age'].fillna(median_age, inplace=True)
# 2. Imputing the missing value in 'Fare' based 
# on the median of the corresponding Pclass
median_fare = titanic_data.groupby('Pclass')['Fare'].median()
titanic_data['Fare'] = titanic_data.apply(
    lambda row: median_fare[row['Pclass']] if pd.isnull(row['Fare']) 
    else row['Fare'], axis=1)
# 3. Filling missing values in 'Carbin' with 'Unknown'
titanic_data['Cabin'].fillna('Unknown', inplace=True)
titanic_data.isnull().sum()
PassengerId    0
Survived       0
Pclass         0
Name           0
Sex            0
Age            0
SibSp          0
Parch          0
Ticket         0
Fare           0
Cabin          0
Embarked       2
dtype: int64
titanic_data['Embarked'].fillna('S', inplace=True)
titanic_data.isnull().sum()
PassengerId    0
Survived       0
Pclass         0
Name           0
Sex            0
Age            0
SibSp          0
Parch          0
Ticket         0
Fare           0
Cabin          0
Embarked       0
dtype: int64

# Descriptive statistics for numerical columns
numerical_stats = titanic_data.describe()
numerical_stats

# Frequency counts for categorical columns
categorical_columns = titanic_data.select_dtypes(include=object).columns
categorical_stats = {column: titanic_data[column].value_counts() 
                     for column in categorical_columns}
categorical_stats
{'Name': Braund, Mr. Owen Harris                     1
 Boulos, Mr. Hanna                           1
 Frolicher-Stehli, Mr. Maxmillian            1
 Gilinski, Mr. Eliezer                       1
 Murdlin, Mr. Joseph                         1
                                            ..
 Kelly, Miss. Anna Katherine "Annie Kate"    1
 McCoy, Mr. Bernard                          1
 Johnson, Mr. William Cahoone Jr             1
 Keane, Miss. Nora A                         1
 Dooley, Mr. Patrick                         1
 Name: Name, Length: 891, dtype: int64,
 'Sex': male      577
 female    314
 Name: Sex, dtype: int64,
 'Ticket': 347082      7
 CA. 2343    7
 1601        7
 3101295     6
 CA 2144     6
            ..
 9234        1
 19988       1
 2693        1
 PC 17612    1
 370376      1
 Name: Ticket, Length: 681, dtype: int64,
 'Cabin': Unknown        687
 C23 C25 C27      4
 G6               4
 B96 B98          4
 C22 C26          3
               ... 
 E34              1
 C7               1
 C54              1
 E36              1
 C148             1
 Name: Cabin, Length: 148, dtype: int64,
 'Embarked': S    646
 C    168
 Q     77
 Name: Embarked, dtype: int64}

# Setting up the figure size and layout fot the historams
plt.figure(figsize=(15, 10))

# The numerical columns to be analyzed
numerical_columns = ['Survived', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']

# Plotting histogram for each numerical column
for i, column in enumerate(numerical_columns):
    plt.subplot(3, 2, i+1)
    sns.histplot(titanic_data[column], kde=False, color='#21a675')
    plt.title(f'Histogram of {column}')
    plt.tight_layout()
plt.show()

# Setting up the figure size and layout for the bar charts
plt.figure(figsize=(12, 5))

# The categorical column to be analyzed
cat_columns = ['Sex', 'Embarked']

# Plotting bar charts for each categorical column
for i, column in enumerate(cat_columns):
    plt.subplot(1, 2, i+1)
    sns.countplot(x=column, data=titanic_data)
    plt.title(f'Bar Chart of {column}')
    plt.tight_layout()
plt.show()

plt.figure(figsize=(18, 6))

# 1. Survived Rate Across Different Classes (Pclass)
plt.subplot(1, 3, 1)
sns.barplot(x='Pclass', y='Survived', data=titanic_data)
plt.title('Survived Rate by Pclass')

# 2. Survived Rate Based on Sex
plt.subplot(1, 3, 2)
sns.barplot(x='Sex', y='Survived', data=titanic_data)
plt.title('Survived Rate by Sex')

# 3. Age vs Fare
plt.subplot(1, 3, 3)
sns.scatterplot(x='Age', y='Fare', hue='Survived', data=titanic_data)
plt.title('Age vs Fare')

plt.tight_layout()
plt.show()

# 1. Average Age and Fare per class (Pclass)
avg_age_fare_per_plcass = titanic_data.groupby('Pclass')[['Age', 'Fare']].mean()

# 2. Survival Rates by Gender
survival_rate_by_gender = titanic_data.groupby('Sex')['Survived'].mean()

avg_age_fare_per_plcass, survival_rate_by_gender
(              Age       Fare
 Pclass                      
 1       36.812130  84.154687
 2       29.765380  20.662183
 3       25.932627  13.675550,
 Sex
 female    0.742038
 male      0.188908
 Name: Survived, dtype: float64)

# 1. Defining a high fare as one that is above the 95th percentile
high_fara_threshold = titanic_data['Fare'].quantile(0.95)
high_fare_data = titanic_data[titanic_data['Fare'] > high_fara_threshold]

# 2. Age Groups with Unusaully High or Low Survival Rates
age_bins = [0, 12, 18, 40, 60, 80]
age_labels = ['Child', 'Teen', 'Adult', 'Middle-Aged', 'Senior']
titanic_data['AgeGroup'] = pd.cut(titanic_data['Age'], bins=age_bins, 
                                  labels=age_labels, right=False)

# Calculating survial rates for each age group
survival_rates_by_age_group = titanic_data.groupby('AgeGroup')['Survived'].mean()

print(f"{high_fare_data[['Fare', 'Pclass', 'Survived', 'Age', 'Sex']]}\n")
print(survival_rates_by_age_group)
Fare  Pclass  Survived    Age     Sex
27   263.0000       1         0  19.00    male
31   146.5208       1         1  28.00  female
88   263.0000       1         1  23.00  female
118  247.5208       1         0  24.00    male
195  146.5208       1         1  58.00  female
215  113.2750       1         1  31.00  female
258  512.3292       1         1  35.00  female
268  153.4625       1         1  58.00  female
269  135.6333       1         1  35.00  female
297  151.5500       1         0   2.00  female
299  247.5208       1         1  50.00  female
305  151.5500       1         1   0.92    male
311  262.3750       1         1  18.00  female
318  164.8667       1         1  31.00  female
319  134.5000       1         1  40.00  female
325  135.6333       1         1  36.00  female
332  153.4625       1         0  38.00    male
334  133.6500       1         1  28.00  female
337  134.5000       1         1  41.00  female
341  263.0000       1         1  24.00  female
373  135.6333       1         0  22.00    male
377  211.5000       1         0  27.00    male
380  227.5250       1         1  42.00  female
390  120.0000       1         1  36.00    male
393  113.2750       1         1  23.00  female
435  120.0000       1         1  14.00  female
438  263.0000       1         0  64.00    male
498  151.5500       1         0  25.00  female
527  221.7792       1         0  28.00    male
557  227.5250       1         0  28.00    male
609  153.4625       1         1  40.00  female
659  113.2750       1         0  58.00    male
660  133.6500       1         1  50.00    male
679  512.3292       1         1  36.00    male
689  211.3375       1         1  15.00  female
700  227.5250       1         1  18.00  female
708  151.5500       1         1  22.00  female
716  227.5250       1         1  38.00  female
730  211.3375       1         1  29.00  female
737  512.3292       1         1  35.00    male
742  262.3750       1         1  21.00  female
763  120.0000       1         1  36.00  female
779  211.3375       1         1  43.00  female
802  120.0000       1         1  11.00    male
856  164.8667       1         1  45.00  female

AgeGroup
Child          0.573529
Teen           0.488889
Adult          0.357724
Middle-Aged    0.394161
Senior         0.240000
Name: Survived, dtype: float64

plt.figure(figsize=(18, 10))

# 1. Survival Rate by Passenger Class (Pclass)
plt.subplot(2, 3, 1)
sns.barplot(x='Pclass', y='Survived', data=titanic_data)
plt.title('Survival Rate by Pclass')

# 2. Survival Rate by Sex
plt.subplot(2, 3, 2)
sns.barplot(x='Sex', y='Survived', data=titanic_data)
plt.title('Survival Rate by Sex')

# 3. Survival Rate by Age Group
plt.subplot(2, 3, 3)
sns.barplot(x='AgeGroup', y='Survived', data=titanic_data)
plt.title('Survival Rate by Age Group')

# 4. Fare vs Survival Rate
plt.subplot(2, 3, 4)
sns.scatterplot(x='Fare', y='Survived', data=titanic_data)
plt.title('Fare vs Survival Rate')

# 5. Survival Rate by Embarkation Point
plt.subplot(2, 3, 5)
sns.barplot(x='Embarked', y='Survived', data=titanic_data)
plt.title('Survival Rate by Embarkation Point')

plt.tight_layout()
plt.show()

from scipy.stats import chi2_contingency


# Creating a contingency table for 'Sex' and 'Survived'
contingency_table = pd.crosstab(titanic_data['Sex'], titanic_data['Survived'])

# Performing the Chi-Square Test
chi2, p_value, _, _ = chi2_contingency(contingency_table)

chi2, p_value
(260.71702016732104, 1.1973570627755645e-58)

import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.utils import resample
# 生成一个不平衡的二分类数据集
X, y = make_classification(n_samples=1000, n_features=10, weights=[0.9, 0.1], random_state=42)
print("原始数据集中，类别0的样本数：", len(y[y==0]))
print("原始数据集中，类别1的样本数：", len(y[y==1]))
原始数据集中，类别0的样本数： 897
原始数据集中，类别1的样本数： 103
# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# 过采样（增加少数类样本）
X_oversampled, y_oversampled = resample(X_train[y_train == 1], y_train[y_train == 1], n_samples=len(X_train[y_train == 0]), random_state=42)
X_oversampled = np.concatenate((X_train[y_train == 0], X_oversampled))
y_oversampled = np.concatenate((y_train[y_train == 0], y_oversampled))

print("过采样后的数据集中，类别0的样本数：", len(y_oversampled[y_oversampled == 0]))
print("过采样后的数据集中，类别1的样本数：", len(y_oversampled[y_oversampled == 1]))
过采样后的数据集中，类别0的样本数： 722
过采样后的数据集中，类别1的样本数： 722
# 欠采样（减少多数类样本）
X_undersampled, y_undersampled = resample(X_train[y_train == 0], y_train[y_train == 0], n_samples=len(X_train[y_train == 1]), random_state=42)
X_undersampled = np.concatenate((X_train[y_train == 1], X_undersampled))
y_undersampled = np.concatenate((y_train[y_train == 1], y_undersampled))

print("欠采样后的数据集中，类别0的样本数：", len(y_undersampled[y_undersampled == 0]))
print("欠采样后的数据集中，类别1的样本数：", len(y_undersampled[y_undersampled == 1]))
欠采样后的数据集中，类别0的样本数： 78
欠采样后的数据集中，类别1的样本数： 78

from imblearn.over_sampling import SMOTE
from sklearn.datasets import make_classification
from collections import Counter
# 创建一个不平衡的二分类数据集作为示例
X, y = make_classification(n_samples=1000, n_features=20, n_informative=2,
                           n_redundant=10, n_clusters_per_class=1, weights=[0.99],
                           flip_y=0, random_state=1)

# 查看原始数据集的类别分布
print('Original dataset shape:', Counter(y))
Original dataset shape: Counter({0: 990, 1: 10})
# 使用SMOTE生成合成样本
smote = SMOTE()
X_resampled, y_resampled = smote.fit_resample(X, y)

# 查看生成的合成样本后的类别分布
print('Resampled dataset shape:', Counter(y_resampled))
Resampled dataset shape: Counter({0: 990, 1: 990})

from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# 创建一个不平衡的二分类数据集作为示例
X, y = make_classification(n_samples=1000, n_features=20, n_informative=2,
                           n_redundant=10, n_clusters_per_class=1, weights=[0.99],
                           flip_y=0, random_state=1)

# 将数据集划分为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# 使用逻辑回归模型，对不同类别的样本定义权重
weights = {0: 1, 1: 10}
clf = LogisticRegression(class_weight=weights)

# 训练模型
clf.fit(X_train, y_train)

# 对测试集进行预测
y_pred = clf.predict(X_test)

# 打印分类报告
print(classification_report(y_test, y_pred))
precision    recall  f1-score   support

           0       1.00      1.00      1.00       297
           1       0.67      0.67      0.67         3

    accuracy                           0.99       300
   macro avg       0.83      0.83      0.83       300
weighted avg       0.99      0.99      0.99       300

from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# 创建一个不平衡的二分类数据集作为示例
X, y = make_classification(n_samples=1000, n_features=20, n_informative=2,
                           n_redundant=10, n_clusters_per_class=1, weights=[0.99],
                           flip_y=0, random_state=1)

# 将数据集划分为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# 使用随机森林算法，建立一个包含10个树的集成模型
clf = RandomForestClassifier(n_estimators=10, random_state=42)

# 训练模型
clf.fit(X_train, y_train)

# 对测试集进行预测
y_pred = clf.predict(X_test)

# 打印分类报告
print(classification_report(y_test, y_pred))
precision    recall  f1-score   support

           0       0.99      1.00      1.00       297
           1       1.00      0.33      0.50         3

    accuracy                           0.99       300
   macro avg       1.00      0.67      0.75       300
weighted avg       0.99      0.99      0.99       300

from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve, f1_score
import matplotlib.pyplot as plt

# 创建一个不平衡的二分类数据集作为示例
X, y = make_classification(n_samples=1000, n_features=20, n_informative=2,
                           n_redundant=10, n_clusters_per_class=1, weights=[0.99],
                           flip_y=0, random_state=1)

# 将数据集划分为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# 使用逻辑回归模型进行训练
clf = LogisticRegression()
clf.fit(X_train, y_train)

# 对测试集进行预测
y_pred_proba = clf.predict_proba(X_test)[:, 1]
y_pred = clf.predict(X_test)

# 计算精确率、召回率和阈值
precision, recall, thresholds = precision_recall_curve(y_test, y_pred_proba)

# 计算F1分数
f1 = f1_score(y_test, y_pred)

# 打印F1分数
print("F1 Score:", f1)

# 绘制精确率-召回率曲线
plt.plot(recall, precision, marker='.')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.show()
F1 Score: 0.8