Sentiment/1_data_preprocessing.py at main · devanshkadu2005/Sentiment · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
import os
import re

import matplotlib.pyplot as plt
import nltk
import numpy as np
import pandas as pd
import seaborn as sns
from datasets import load_dataset
from nltk.corpus import stopwords
from textblob import TextBlob

nltk.download('stopwords')
nltk.download('punkt')

os.makedirs('results/preprocessing', exist_ok=True)

# Load Dataset
# ------------------------------------------------------------
dataset = load_dataset('go_emotions', 'simplified')
train_df = pd.DataFrame(dataset['train'])
val_df = pd.DataFrame(dataset['validation'])
test_df = pd.DataFrame(dataset['test'])

EMOTION_NAMES = [
    'admiration', 'amusement', 'anger', 'annoyance', 'approval',
    'caring', 'confusion', 'curiosity', 'desire', 'disappointment',
    'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear',
    'gratitude', 'grief', 'joy', 'love', 'nervousness', 'optimism',
    'pride', 'realization', 'relief', 'remorse', 'sadness', 'surprise', 'neutral'
]

SIMPLIFIED = {
    'joy': [
        'admiration', 'amusement', 'approval', 'caring', 'desire',
        'excitement', 'gratitude', 'joy', 'love', 'optimism', 'pride', 'relief'
    ],
    'sadness': ['disappointment', 'grief', 'remorse', 'sadness'],
    'anger': ['anger', 'annoyance', 'disapproval', 'disgust'],
    'fear': ['embarrassment', 'fear', 'nervousness'],
    'surprise': ['confusion', 'curiosity', 'realization', 'surprise'],
    'neutral': ['neutral']
}


def map_label(label_list):
    if not label_list:
        return 'neutral'

    # take first label
    idx = label_list[0]
    name = EMOTION_NAMES[idx]

    for emotion, members in SIMPLIFIED.items():
        if name in members:
            return emotion

    return 'neutral'


for df in [train_df, val_df, test_df]:
    df['emotion'] = df['labels'].apply(map_label)

# Text Cleaning
# ------------------------------------------------------------
STOPWORDS = set(stopwords.words('english'))


def clean_text(text):
    text = str(text).lower()
    text = re.sub(r'http\S+', '', text)
    text = re.sub(r'[^a-z\s]', '', text)
    tokens = text.split()
    tokens = [t for t in tokens if t not in STOPWORDS and len(t) > 2]
    return ' '.join(tokens)


for df in [train_df, val_df, test_df]:
    df['clean_text'] = df['text'].apply(clean_text)

# Structural Features
# ------------------------------------------------------------
def extract_features(text):
    blob = TextBlob(text)
    words = text.split()
    return {
        'word_count': len(words),
        'char_count': len(text),
        'avg_word_length': np.mean([len(w) for w in words]) if words else 0,
        'exclamation_count': text.count('!'),
        'question_count': text.count('?'),
        'uppercase_ratio': sum(1 for c in text if c.isupper()) / max(len(text), 1),
        'punctuation_ratio': sum(1 for c in text if c in '.,;:!?') / max(len(text), 1),
        'sentiment_polarity': blob.sentiment.polarity,
        'sentiment_subjectivity': blob.sentiment.subjectivity,
    }


for df in [train_df, val_df, test_df]:
    feats = df['text'].apply(extract_features).apply(pd.Series)
    for col in feats.columns:
        df[col] = feats[col]

# Save
# ------------------------------------------------------------
train_df.to_csv('results/preprocessing/train_processed.csv', index=False)
val_df.to_csv('results/preprocessing/val_processed.csv', index=False)
test_df.to_csv('results/preprocessing/test_processed.csv', index=False)

# Plots
# ------------------------------------------------------------
# Label distribution
plt.figure(figsize=(10, 5))
train_df['emotion'].value_counts().plot(kind='bar', color='steelblue')
plt.title('Emotion Distribution in Training Set')
plt.xlabel('Emotion')
plt.ylabel('Count')
plt.tight_layout()
plt.savefig('results/preprocessing/emotion_distribution.png')
plt.close()

# Feature distributions
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
feats_to_plot = [
    'word_count', 'sentiment_polarity', 'sentiment_subjectivity',
    'exclamation_count', 'uppercase_ratio', 'punctuation_ratio'
]
for ax, feat in zip(axes.flatten(), feats_to_plot):
    for emo in train_df['emotion'].unique():
        subset = train_df[train_df['emotion'] == emo][feat]
        subset.hist(ax=ax, alpha=0.4, label=emo, bins=30)
    ax.set_title(feat)
    ax.legend(fontsize=6)

plt.suptitle('Feature Distributions by Emotion')
plt.tight_layout()
plt.savefig('results/preprocessing/feature_distributions.png')
plt.close()

print('Preprocessing done.')
print(f'Train: {len(train_df)} | Val: {len(val_df)} | Test: {len(test_df)}')
print(train_df['emotion'].value_counts())