!pwd

!head -n 2 data/dataset.csv*

## Install python3.7 and above for the codes to work

## Load the python modeules

import os
import sys
import logging
from dataclasses import dataclass

import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

import seaborn as sns
sns.set_context("poster")
sns.set(rc={'figure.figsize': (16, 9.)})
#sns.set_style("whitegrid")

import plotly_express as px

import numpy as np
import pandas as pd
pd.set_option('display.max_colwidth', -1)

logging.basicConfig(level=logging.INFO, stream=sys.stdout)

#Identify the file in folder
for file in os.listdir(os.curdir):
    if file.startswith('zendesk'):
        print(f'csv file needed: {file}')

df = pd.read_csv('data/dataset.tsv', delimiter='\t',encoding='utf-8')

df.sample(5)

f'2. Total number of observations:  {len(df)}'

print(f'3. Are any null in the features? \n\n {df.isnull().sum()}')

f'4. How many of observations that are dublicates?: {df.duplicated().sum()}'

print('5. Number of dublicates and unique values per feature')
print('----------------------------------------------------')
def determine_number_duplicated(df:pd.DataFrame, col:str='QuestionID')-> str:
    '''
    The function prints out the number of duplicate in the dataframe features

    Parameters:
    ---------
    df: DataFrame
    col: str

    print: str

    '''
    print(f'The number of duplicate and unique value in {col} is {df.duplicated(subset=[col]).sum()} and {df[col].nunique()}' )

for col in df.columns:
    determine_number_duplicated(df, col)

print(df.groupby(['Question'])['SentenceID'].count().sort_values(ascending=False).head(1))
print(df.groupby(['Question'])['SentenceID'].count().sort_values(ascending=False).tail(1))

print(df.groupby(['QuestionID', 'Sentence'])['SentenceID'].count().sort_values(ascending=False).head(5))

##Filter out the two questionIds (Q1368 and Q1029) in the dataset
cm = sns.light_palette("green", as_cmap=True)

df[(df['QuestionID']=='Q1368') | (df['QuestionID']=='Q1029')].loc[[7328,7329,9637, 9641], :]

##Remove two rows from index 7328 and 9637 from dataframe
new_df = df.copy()
new_df.drop([7328, 9637], inplace=True)

new_df[(new_df['QuestionID']=='Q1368') | (new_df['QuestionID']=='Q1029')]

### Distribution of Label Feature
new_df["Label"].value_counts().plot(kind='bar', title="Chart 1: Distribution of Label Feature")

##Distribution of questions with number of Sentences

new_df["QuestionID"].value_counts().plot(kind='hist', title="Chart 2: Distribution of questions with number of Sentences")

##Distribution of questions with number of Sentences

new_df.groupby(["DocumentTitle"])['SentenceID'].count().plot(kind='hist', title="Chart 3: Distribution of DucumentTitle with number of Sentences")
print(' ')
new_df.groupby(["DocumentTitle"])['SentenceID'].count().sort_values(ascending=False).head(10)

new_df[new_df['QuestionID']=='Q1']

from typing import *


@dataclass
class DataProcessing:
    '''
    This class will have different methods for data processing
    '''
    df: pd.DataFrame
    feature1: str
    feature2: str
    feature3: str

    def question_with_no_answer(self) -> List[str]:
        '''
        The function will take argument feature and use dataframe provided its class.
        Parameters:
        ----------


        return
        ------
        List of strings

        '''
        qna_df = self.df.groupby([self.feature1, self.feature2])[self.feature3].count().reset_index()
        qna_df = qna_df.pivot(index=self.feature1, columns=self.feature2, values=self.feature3).reset_index()
        qna_null_df = qna_df[qna_df[1].isnull()]
        return  list(qna_null_df[self.feature1])

    def questions_with_answer(self) -> List[str]:
        '''
        The function will take no argument feature and use dataframe and feature  provided its class.
        Parameters:
        ----------


        return
        ------
        List of strings
        '''
        question_list = self.question_with_no_answer()
        list_of_questions = set([item for item in self.df[self.feature1] if item not in question_list])
        return list_of_questions

    def question_df(self, question_list:List[str]) -> pd.DataFrame:
        '''
        The function will take argument list of strings and use dataframe and features provided its class.
        Parameters:
        ----------

        return
        ------
        pd.Dataframe
        '''
        list_of_dfs = [self.df[self.df[self.feature1]==item] for item in question_list]
        question_df = pd.concat( list_of_dfs, axis=0)
        return  question_df

    def consolate_features(self) -> pd.DataFrame:
        '''
        function join two Series into one and return a new dataframe
        return
        -------
        pd.Dataframe
        '''
        feature = '_'.join([self.feature1, self.feature3])
        df[feature] = self.df[self.feature1] +' '+ self.df[self.feature3]
        return df


    def text_with_longest_length(self, s: pd.Series) -> dict:
        '''
        '''
        result = {}
        max_length = 0
        min_length= 5
        max_text = None
        min_text = None
        for item in s:
            length =len(str(item).split(' '))
            if length > max_length:
                max_length = length
                max_text = item
        #result['max'] = [max_text, max_length]
        return [max_text, max_length]

if __name__ == '__main__':
    DataPro = DataProcessing(new_df, 'QuestionID', 'Label', 'SentenceID')
    question_with_no_answer=DataPro.question_with_no_answer()
    print(f'how many unique observable questions without answers: {len(question_with_no_answer)}')
    questions_with_answer = DataPro.questions_with_answer()
    print(f'how many unique observable questions with answers: {len(set(questions_with_answer))}')
    question_df = DataPro.question_df(questions_with_answer)
    print(f'how many observable questions in the question with answers dataframe: {len(question_df)}')
    print( f'Max length of questions; {DataPro.text_with_longest_length(question_df["Question"])[1]}')

## new dataframe
question_df.head()

question_df['Label'].value_counts().plot(kind='bar', title="Chart 4: Distribution of new Label ")

from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.text import Tokenizer, text_to_word_sequence
from tensorflow.keras.preprocessing.sequence import pad_sequences

from tensorflow.keras.models import Sequential
from tensorflow.keras import layers

class FeatureEngineering(DataProcessing):
    '''
    Parameters:
    ----------
    df: pd.DataFrame
    feature1: str
    feature2: str
    feature3: str
    '''

    def __init__(self, df: pd.DataFrame, feature1:str, feature2:str, feature3:str):
        super().__init__(df, feature1, feature2, feature3)

    #feature = '_'.join([self.feature1, self.feature3])

    def join_features(self) -> pd.DataFrame:
        '''
        Function will return dataframe
        Return
        ------
        pd.DataFrame
        '''
        result = super().consolate_features()
        feature = '_'.join([self.feature1, self.feature3])
        return result[[feature, self.feature2]]

    def determine_maxlen(self) -> List[int]:
        '''
        The function will return list
        Return
        ------
        List
        '''
        feature = '_'.join([self.feature1, self.feature3])
        s = self.join_features()[feature]
        #print(s)
        result = super().text_with_longest_length(s)
        return result[1]


    def dataset_split(self) -> pd.DataFrame:
        '''
        The function will return four dataframes
        Return
        ------
        pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame,
        '''
        feature = '_'.join([self.feature1, self.feature3])
        new_df = self.join_features()
        x_train,x_test, y_train,y_test = train_test_split(new_df[feature], new_df[self.feature2], test_size=0.25, random_state=0)
        return x_train,x_test, y_train,y_test

    def text_tokenize(self, text:pd.Series, num_words:int=None):
        '''
        The function will return function
        Return
        ------
        func
        '''
        tokenizer = Tokenizer(num_words=num_words, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',lower=True)
        tokenizer.fit_on_texts(text)
        return tokenizer

    def tokenize_sequence(self, text:pd.Series, func, maxlen:int, padding:str="post") -> List[list]:
        '''
        The function will return matrix
        Return
        ------
        Matrix
        '''
        new_text=func.texts_to_sequences(text)
        new_text= pad_sequences(new_text, padding=padding, maxlen=maxlen)
        return new_text

#Create the embedding layer
from dataclasses import dataclass

@dataclass
class QuestionAnswerClassifer:
    input_dim: int
    output_dim:int
    input_length:int

    def create_model(self):
        model = Sequential()
        model.add(layers.Embedding(input_dim=self.input_dim, output_dim=self.output_dim, input_length=self.input_length))
        model.add(layers.Flatten())
        model.add(layers.Dense(64, activation='relu'))
        model.add(layers.Dense(128, activation='relu'))
        model.add(layers.Dense(1, activation='sigmoid'))
        return model

    def compile_model(self):
        model = self.create_model()
        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
        return model

    def plot_history(self, history):
        accuracy = history.history['accuracy']
        val_accuracy = history.history['val_accuracy']
        loss = history.history['loss']
        val_loss = history.history['val_loss']
        x = range(1, len(accuracy) + 1)
        plt.figure(figsize=(12, 5))
        plt.subplot(1, 2, 1)
        plt.plot(x, accuracy, 'b', label='Training accuracy')
        plt.plot(x, val_accuracy, 'r', label='Validation accuracy')
        plt.title('Training and validation accuracy')
        plt.legend()
        plt.subplot(1, 2, 2)
        plt.plot(x, loss, 'b', label='Training loss')
        plt.plot(x, val_loss, 'r', label='Validation loss')
        plt.title('Training and validation loss')
        plt.legend()

import sys
import warnings

import mlflow
import mlflow.keras
import mlflow.tensorflow


if __name__ == '__main__':

    Feature_Engineering = FeatureEngineering(new_df, 'Question', 'Label','Sentence')

    maxlen =  Feature_Engineering.determine_maxlen() + 10
    print(f'Maxlen:{maxlen}')

    #Split the dataset into train and test set
    question_df['Question_Sentence'] = question_df['Question'] +' '+ question_df['Sentence']
    features = question_df['Question_Sentence']
    target = question_df['Label']
    x_train, x_test, y_train, y_test = train_test_split(features, target, test_size=0.25, random_state=0)

    tokenizer = Feature_Engineering.text_tokenize(x_train.values, num_words=5000)

    vocab_size = len(tokenizer.word_index) + 1

    xtrain= Feature_Engineering.tokenize_sequence(x_train, tokenizer, maxlen=maxlen)  
    xtest = Feature_Engineering.tokenize_sequence(x_test, tokenizer, maxlen=maxlen)  

    embedding_dim = 100 #int(sys.argv[1]) if len(sys.argv) > 1 else 50
    epochs = 20 #int(sys.argv[2]) if len(sys.argv) > 1 else 20
    batch_size = 10 #int(sys.argv[3]) if len(sys.argv) > 3 else 10

    with mlflow.start_run():
        cls = QuestionAnswerClassifer(input_dim=vocab_size, output_dim=embedding_dim, input_length=maxlen)

        model = cls.compile_model()

        model.summary()
        print(' ')

        history = model.fit(xtrain, y_train,
                    epochs=epochs,
                    verbose=False,
                    validation_data=(xtest, y_test),
                    batch_size=batch_size)

        print('Evaluation of model accuracy for trainingset')
        train_loss, train_accuracy = model.evaluate(xtrain, y_train)
        print(' ')
        print('Evaluation of model accuracy for testset')
        test_loss, test_accuracy = model.evaluate(xtest, y_test)
        print(' ')
        cls.plot_history(history)

        #log the metrics and parameters
        mlflow.log_param('embedding', embedding_dim)
        mlflow.log_param('epochs', epochs)
        mlflow.log_param('batch_size', batch_size)
        mlflow.log_metric("train_loss", train_loss)
        mlflow.log_metric("train_accuracy", train_accuracy)
        mlflow.log_metric("test_loss", test_loss)
        mlflow.log_metric("test_accuracy", test_accuracy)


        ## The function in MLflow saved the model as an artifact within the run.
        mlflow.keras.log_model(model, "model")