# -*- coding: utf-8 -*-
"""
Created on Mon Oct 17 09:06:57 2022

@author: Márton
"""

import numpy as np;
from sklearn.datasets import fetch_20newsgroups;
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer;
from sklearn.linear_model import LogisticRegression;


categories = [
        'comp.graphics',
        'comp.os.ms-windows.misc',
        'comp.sys.ibm.pc.hardware',
        'comp.sys.mac.hardware',
        'comp.windows.x',
        'sci.crypt',
        'sci.electronics',
        'sci.med',
        'sci.space'
];

ds_train = fetch_20newsgroups(subset='train',categories=categories);
ds_test = fetch_20newsgroups(subset='test',categories=categories);
n_train = len(ds_train.data);
n_test = len(ds_test.data);

target_names = ['computer','science'];
target_train = np.where(ds_train.target>4,1,0);
target_test = np.where(ds_test.target>4,1,0);

Tfidf_vector = TfidfVectorizer(stop_words='english',max_df=0.8,min_df=0.01); 
DT_train_tfidf = Tfidf_vector.fit_transform(ds_train.data); 
vocabulary_tfidf = Tfidf_vector.get_feature_names();
n_words = DT_train_tfidf.shape[1];

# document-term matrix in dense form 
DT_train_tfidf_dense = DT_train_tfidf.toarray();

clf = LogisticRegression();
clf.fit(DT_train_tfidf,target_train);
accuracy = clf.score(DT_train_tfidf,target_train);