from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.linear_model import SGDClassifier
import numpy as np

# Setup for streaming processing
vectorizer = HashingVectorizer(n_features=10000)
classifier = SGDClassifier(loss='log_loss', alpha=0.01)

# Simulate streaming data
categories = ['alt.atheism', 'soc.religion.christian', 
              'comp.graphics', 'sci.med']
newsgroups = fetch_20newsgroups(subset='train', categories=categories)

# Process in batches
batch_size = 100
n_batches = len(newsgroups.data) // batch_size

for i in range(n_batches):
    start_idx = i * batch_size
    end_idx = (i + 1) * batch_size

    # Get batch
    batch_docs = newsgroups.data[start_idx:end_idx]
    batch_labels = newsgroups.target[start_idx:end_idx]

    # Vectorize
    X_batch = vectorizer.transform(batch_docs)

    # Incremental learning
    classifier.partial_fit(X_batch, batch_labels, 
                          classes=np.unique(newsgroups.target))

        __
        __