Cars_Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4614469d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import os \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "import re\n",
    "import numpy as np\n",
    "from scipy.stats import zscore\n",
    "import seaborn as sns\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from statistics import mode\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "import statsmodels.api as sm\n",
    "import arabic_reshaper\n",
    "from bidi.algorithm import get_display\n",
    "import matplotlib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "import pickle\n",
    "from transformation_functions import square_root_transformation , log_transformation , Reciprocal_transformation\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d6e06ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_path = 'C:/Users/IMOE001/Downloads/data/data'\n",
    "folder_path2 = 'C:/Users/IMOE001/Downloads/data/test' #small sample for testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a26c8541",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def extract_features(html_code):\n",
    "    soup = BeautifulSoup(html_code, 'html.parser') # object of html code \n",
    "    features = {}\n",
    "    \n",
    "    car_info = soup.find('td', width='65%') # the first matching only \n",
    "    if car_info:\n",
    "        features['اسم السيارة'] = car_info.find('h3').get_text(strip=True)  \n",
    "        features['موديل السيارة'] = car_info.find('h5').get_text(strip=True)  \n",
    "    \n",
    "    car_price = soup.find('h5', class_='post-price')\n",
    "    if car_price:\n",
    "        for a_tag in car_price.find_all('a'): # return a list \n",
    "            a_tag.extract()\n",
    "        features['سعر السيارة'] = car_price.get_text(strip=True) if car_price else None \n",
    "    \n",
    "    car_characteristics = soup.find_all('tr', class_='list-row')\n",
    "    for characteristic in car_characteristics:\n",
    "        featurename_and_itsvalue = characteristic.find_all('td')\n",
    "        feature_name = featurename_and_itsvalue[0].get_text(strip=True)\n",
    "        feature_value = featurename_and_itsvalue[1].get_text(strip=True)\n",
    "        features[feature_name] = feature_value    \n",
    "    \n",
    "    additions = soup.find('td', valign='top', text='إضافات')\n",
    "    if additions:\n",
    "        ul_element = additions.find_next('ul')\n",
    "        li_elements = ul_element.find_all('li')\n",
    "\n",
    "        list_of_additions = [li.get_text(strip=True) for li in li_elements if li.get_text(strip=True) != 'فتحة سقف']\n",
    "        print(list_of_additions)\n",
    "        num_additions = len(list_of_additions)\n",
    "        features['Number_of_Additions'] = num_additions\n",
    "\n",
    "        features['فتحة السقف'] = 1 if 'فتحة سقف' in [li.get_text(strip=True) for li in li_elements] else 0\n",
    "\n",
    "\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd4dcefd",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = [] \n",
    "for filename in os.listdir(folder_path):\n",
    "    file_path =folder_path  + '/' + filename\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        html_code = file.read()\n",
    "        data_of_one_sample = extract_features(html_code)\n",
    "        print(f\" Sample: {data_of_one_sample}\")\n",
    "        all_data.append(data_of_one_sample)\n",
    "\n",
    "#all features => list of dictionarys "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "642d3b47",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = pd.DataFrame(all_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580c00f0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a7be370",
   "metadata": {},
   "outputs": [],
   "source": [
    "#total_rows_with_missing_values =cars_data.isnull().all(axis=1).sum()\n",
    "#total_rows_with_missing_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9165cc88",
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_file_path = 'C:\\\\Users\\\\IMOE001\\\\Desktop\\\\cars_data.csv'\n",
    "cars_data.to_csv(csv_file_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78025b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = pd.read_csv('cars_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20f85eac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cars_data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a99a4efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = cars_data.drop(columns= 'إضافات')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8def551",
   "metadata": {},
   "source": [
    "# 21 features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb5a73e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "625fc634",
   "metadata": {},
   "source": [
    "# 6988  samples "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec6ea9cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70f745dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_rows_with_missing_values = cars_data.isnull().all(axis=1).sum()\n",
    "total_rows_with_missing_values "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d8faa8e",
   "metadata": {},
   "source": [
    "This mean 827 html page dont contain required data about cars "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad343c7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_columns_with_missing_values = cars_data.isnull().all(axis=0).sum()\n",
    "total_columns_with_missing_values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f7023c9",
   "metadata": {},
   "source": [
    "# drop the rows with missing values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dc6e1ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = cars_data.dropna(how='all')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8df12ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reindex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d66041c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_values_in_the_features = cars_data.isnull().sum()\n",
    "missing_values_in_the_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8466d26a",
   "metadata": {},
   "source": [
    "# The feature with the most missing values 'الدفع'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec414dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop الدفع feature \n",
    "cars_data = cars_data.drop(columns='الدفع')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "462cf109",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98a683f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "maping_description = {\n",
    "    \n",
    "    'اسم السيارة' : 'Name',\n",
    "    \"لون السيارة\": 'color',\n",
    "    \"نوع الوقود\": 'fuel_type',\n",
    "    \"أصل السيارة\": 'origin_car',\n",
    "    \"عداد السيارة\": 'car_speedometer',\n",
    "    \"أصحاب سابقون\": 'ex_owners',\n",
    "    \"رخصة السيارة\": 'car_license',\n",
    "    \"نوع الجير\": 'lime_type',\n",
    "    \"الزجاج\": 'glass',\n",
    "    \"قوة الماتور\": 'motor_power',\n",
    "    \"عدد الركاب\": 'passengers',\n",
    "    'فتحة السقف': 'sunroof',\n",
    "    'موديل السيارة' : 'model',\n",
    "    'سعر السيارة': 'Price',\n",
    "\n",
    "}\n",
    "\n",
    "# Create a new DataFrame with updated column names\n",
    "cars_data= cars_data.rename(columns=maping_description)\n",
    "\n",
    "# Display the new DataFrame\n",
    "cars_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fbb9fd3",
   "metadata": {},
   "source": [
    "# عداد السيارة , عدد الركاب , اصحاب سابقون \n",
    "# The only features with missing values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9e6bafd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_types = cars_data.dtypes\n",
    "data_types"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ff2db2",
   "metadata": {},
   "source": [
    "# The numeric features are [motor_power , Number_of_Additions, sunroof]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a02a5d9",
   "metadata": {},
   "source": [
    "# The status of the ad does not affect the price of the car"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5e61fbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop features related to ad\n",
    "ad_columns = ['حالة الإعلان' , 'تاريخ نشر الإعلان' , 'تاريخ إنتهاء الإعلان']\n",
    "\n",
    "cars_data = cars_data.drop(columns= ad_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeb77867",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cars_data = cars_data.drop(columns= ['معروضة' , 'وسيلة الدفع'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01183b3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0b61cd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce9ac9f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define your mapping dictionary\n",
    "value_mapping = {\n",
    "    \n",
    "    'يد اولى': 1,\n",
    "    'أولى': 1,\n",
    "    'يد اولا':1 , \n",
    "    'يد أولى' : 1 ,\n",
    "    'يد أولى استيراد' : 1 , \n",
    "    'يد اولي ' : 1 , \n",
    "    'واحد' : 1 , \n",
    "    'اولا' : 1 ,\n",
    "    'يد 1' : 1 , \n",
    "    '١' : 1 , \n",
    "    'يداولى' : 1 , \n",
    "    'يد أولى وكالة' : 1 , \n",
    "    'يدو اولى': 1 ,\n",
    "    'يد واله' :1 ,\n",
    "    'اول':1 , \n",
    "    'يد ولا' : 1 , \n",
    "    'يد اولى شركة' : 1 , \n",
    "    'يد. اولى' : 1 , \n",
    "    'ياولى' : 1 , \n",
    "    'يد اولى وكالة' : 1 , \n",
    "    'يد اولي' : 1 , \n",
    "    'يدى اوله' : 1 , \n",
    "    'يد اولىً' : 1 , \n",
    "    'يد واحد' : 1 , \n",
    "    'يد اةلي' : 1 , \n",
    "    'يد اولى الماني':1 ,\n",
    "    'يد  ١' : 1 , \n",
    "    '01' : 1 ,\n",
    "    'يد اولة' : 1 , \n",
    "    'ايد أولى' : 1 , \n",
    "    'يد اولى استيراد شخصي':1 , \n",
    "    'يد ا': 1 ,\n",
    "    'يد اوله' : 1 , \n",
    "    'يداولى سحب شركه':1 , \n",
    "    'اولى' : 1 , \n",
    "    '1' : 1 ,\n",
    "    'يد اول' : 1 ,\n",
    "    'يدأولى' : 1 , \n",
    "    'يد' : 1 ,\n",
    "    'يد ١' : 1 , \n",
    "    'يد واحدة' : 1 ,\n",
    "    'يدأولى' : 1 , \n",
    "    'يد اولى (شركة)':1 ,\n",
    "    'يد اولى اصحاب سابقون 0':1 , \n",
    "    'يد اولى  اصل شركة':1 ,\n",
    "    'وارد كوريا يد اولى' : 1 , \n",
    "    'يد أولئ أصحاب سابقون صفر' : 1 , \n",
    "    'يد اولى صفر كيلو' : 1 , \n",
    "    'اوله'  : 1 , \n",
    "    'يداولا' : 1 ,\n",
    "    '00' : 0 ,\n",
    "    '0' : 0 , \n",
    "    '0000' : 0 , \n",
    "    '...' : 0  , \n",
    "    '......' : 0 , \n",
    "    '0 مستورده' : 0 , \n",
    "    '-' : 0 ,\n",
    "    '00000' : 0 , \n",
    "    'استيراد مباشر من كوريا' : 0 ,\n",
    "    '********' : 0 ,\n",
    "    '٠' : 0 ,\n",
    "    '0مستورد' : 0 , \n",
    "    '.' : 0 , \n",
    "    '.........' : 0 ,\n",
    "    '00000000' : 0 , \n",
    "    '.........' : 0 ,\n",
    "    '٠٠٠٠٠٠٠٠٠٠' : 0 , \n",
    "    'مش منمر . (صفر)' : 0 ,\n",
    "     '---' : 0 , \n",
    "    'Zero' : 0 ,\n",
    "    '٠٠٠٠٠٠٠' : 0 , \n",
    "    'غير منمرمستوردة' : 0 , \n",
    "    '..' : 0 , \n",
    "    'استيراد كوري' : 0,\n",
    "    '0 0000000' : 0 ,\n",
    "    'ستيراد المانيا' : 0 , \n",
    "    'استيراد كوريا - جديد مش منمر' : 0 , \n",
    "    'ستوردة غير منمرة' : 0 , \n",
    "   'استيراد مباشر' : 0 ,\n",
    "    'اصل شركة' : 0 , \n",
    "    'يد أولى،...' : 1 , \n",
    "    'Ooo' : 0 , \n",
    "    '0000000' : 0 ,\n",
    "    'يد أولى 1' : 1 , \n",
    "    'يد أولي' : 1 , \n",
    "   'يد أولئ' : 1 , \n",
    "    'يد أولى بعد الاستيراد' : 1 , \n",
    "    'ا' : 1 , \n",
    "    'مستورده يد اولى' : 1 , \n",
    "    'مستوردة غير مرخصة' : 0 ,\n",
    "    'مستود غير منمر' : 0 , \n",
    "    'اصحاب سابقون صفر' : 0 , \n",
    "    'استراد شخصي' : 0 , \n",
    "    '0 مستورد كوري' : 0 ,\n",
    "     '******' : 0 ,\n",
    "    'لا يوجد' : 0 , \n",
    "    'مستورد شركة' : 0 , \n",
    "    'استيراد'  : 0 , \n",
    "    '000000' : 0 , \n",
    "    '**********' : 0 ,\n",
    "   'من الشركه مباشر لا يوجد اصحاب سابقون' : 0 ,\n",
    "    'استيراد مباشر من اوروبا' : 0 ,\n",
    "    'ايد اولى' : 1 ,\n",
    "    'فقط يد اولى' : 1 , \n",
    "    '٠٠٠٠' : 0 ,\n",
    "    'لا شيء' : 0 , \n",
    "    'يد 0' : 0 , \n",
    "    '،' : 0 , \n",
    "    '....' : 0 ,\n",
    "    '٠٠٠٠٠٠٠٠' : 0 , \n",
    "    '٠٠٠٠٠٠٠٠٠' : 0 , \n",
    "    '٠٠٠٠٠٠' : 0 ,\n",
    "    '٠٠' : 0 , \n",
    "    '٠٠٠٠٠٠٠٠' : 0 ,\n",
    "    'سحب شركة' : 0 ,\n",
    "    'يد اولى، استيراد شخصي' : 1 , \n",
    "    'يد اولى فقط' : 1 , \n",
    "    'يد أولئ':1 , \n",
    "   'مستورده' : 0 , \n",
    "    '، 0' : 0 , \n",
    "    '٠٠٠٠٠' : 0 ,\n",
    "    '0استيراد' : 0 ,\n",
    "    'لايوجد' : 0 ,\n",
    "   'يد اولى فقط' : 1 , \n",
    "    'اولى\\x89' : 1 , \n",
    "    'يد ولى' : 1 , \n",
    "    'مستورد' : 0 ,\n",
    "    'يد صفر' : 0 ,\n",
    "    'استيراد الماني' : 0 ,\n",
    "    '0 استيراد كوري' : 0 , \n",
    "    'استيراد المانيا' : 0 , \n",
    "    'غير منمر استيراد الماني' : 0 , \n",
    "    'صفر' : 0 ,\n",
    "    '000' : 0 ,\n",
    "'مستوردة' : 0 ,   \n",
    "   'استراد' : 0 ,\n",
    "    'استيراد كوريا': 0 ,\n",
    "    'وارد المانيا' : 0 ,\n",
    "    'أستيراد ألماني' : 0 , \n",
    "    'مستوردة غير منمرة' : 0 ,\n",
    "    'O' : 0 , \n",
    "    'يد ثانيه' : 2 ,\n",
    "    'يد ثانية' : 2 ,\n",
    "    'ثانية':2,\n",
    "    'يد  ثانية':2, \n",
    "    'تانية': 2 ,\n",
    "    'ايدثانيه':2,\n",
    "    'تانبه' : 2 ,\n",
    "    'يد اولئ':1 ,\n",
    "    'يد تاني':2,\n",
    "    'ثانبه':2 , \n",
    "    'يد تاني':2 ,\n",
    "    'أيد تانية':2,\n",
    "    'يد ٢':2,\n",
    "    'يد ثني': 2,\n",
    "    'ايد ثانيه':2,\n",
    "    'ايد ثانية':2,\n",
    "    'يد ثاني':2 ,\n",
    "    '2':2 ,\n",
    "    '٢':2 ,\n",
    "    'يد 2':2 ,\n",
    "    'يد اولى ( شركة )': 1 ,\n",
    "    'يدثانيه':2 ,\n",
    "    'ثانيه':2,\n",
    "    'اثنان':2 ,\n",
    "    'يد تانيه':2,\n",
    "    'يد ثاتيه':2,\n",
    "    '2يد':2,\n",
    "    'ثاني':2,\n",
    "    'يد2':2,\n",
    "    'صفرض':0,\n",
    "    'يدتانيه':2,\n",
    "    'يد ثاني.':2,\n",
    "    'صفرر':0,\n",
    "    'غير منمره':0,\n",
    "    'غير منمرة':0,\n",
    "    'مش منمر':0,\n",
    "    'تاني':2,\n",
    "    'يد ثالثة':3,\n",
    "    'ثالثة':3,\n",
    "    'مش منمرة':0,\n",
    "    '4':4,\n",
    "    '3':3,\n",
    "    'اربعة':4,\n",
    "    '٣':3,\n",
    "    '7':7,\n",
    "    'رابعه':4,\n",
    "    '٤':4,\n",
    "    'يد 4':4,\n",
    "    'يد خامسة':5,    \n",
    "    'يد رابعه':4,\n",
    "    'يد ثالثة ( فتيات)':3,\n",
    "    'يد رابعة':4,\n",
    "    '5': 5  ,\n",
    "    '9' : 9 ,\n",
    "    'يد تالته':3,\n",
    "    'يد التالته':3,\n",
    "    'تالته':3,\n",
    "    'يد خامساً':5,\n",
    "    'يد 5':5,\n",
    "    'يد سادسة':6,\n",
    "    'يد ثالثه':3,\n",
    "    '6':6,\n",
    "    'يد ٤ بس نخب السيارة':4,\n",
    "    'مش منمره':0,\n",
    "    'يدثالثة':3,\n",
    "    'يد خامسه':5,\n",
    "    'يد4':4,\n",
    "    'يد 3':3,\n",
    "    'ايد ثالثة':3,\n",
    "    'ثمانيه':8,\n",
    "    'يد 6':6,\n",
    "    'غير منمر':0,\n",
    "    'يد سابعه':7,\n",
    "    'يد٢':2,\n",
    "    'خمسه':5,\n",
    "    'يد خمسة':5,\n",
    "    'يد 8':8,\n",
    "    'يد تانية':2,\n",
    "    'يدثانية':2,\n",
    "    'تانيه':2,\n",
    "    '٨':8,\n",
    "    '٦':6,\n",
    "    'رابعة':4,\n",
    "    'ثالثه':3,\n",
    "    '8':8,\n",
    "    '٧':7,\n",
    "    '٩':9,\n",
    "    '٥':5,\n",
    "    'مش منمرا':0,\n",
    "    'انا':1,\n",
    "    'يدخامسه':5,\n",
    "    'يد٤':4,\n",
    "    'يد ٣':3,\n",
    "    'يد٥':5,\n",
    "    'عدد٣':3,\n",
    "    'يد٣':3,\n",
    "    'ايد ثالثه':3,\n",
    "    'تالثه':3,\n",
    "    'ثالثه':3,\n",
    "    'يدثالثه':3,\n",
    "    'يد ثامنه':8,\n",
    "    'خمس':5,\n",
    "    'خامسة':5,\n",
    "    'يد٥':5,\n",
    "    'يد ٥':5,\n",
    "    'ثالث':3,\n",
    "    'يد ٤':4,\n",
    "    'يدخامسه':5,\n",
    "    '04':4,\n",
    "    'يد اربعه':4,\n",
    "    'يد ٧':7,\n",
    "    'يديانيه':2,\n",
    "    'يد6':6,\n",
    "    '13':13,\n",
    "    '١٠':10,\n",
    "    '16':16,\n",
    "    '17':17,\n",
    "    '13':13,\n",
    "    '١١':11,\n",
    "    '03':3,\n",
    "    ',3':3,\n",
    "    '18':18,\n",
    "    '15':15,\n",
    "    'عدد٣':3,\n",
    "    '12':12,\n",
    "    '11':11,\n",
    "    '10':10,\n",
    "    'خمسة':5,\n",
    "    'يد ثانيا':2,\n",
    "    '14':14,\n",
    "    'يد سابعة':7,\n",
    "    'ايد رابعه':4,\n",
    "    'خمسة':5,\n",
    "    '١٨':18,\n",
    "    'ايد 2':2,\n",
    "    '١٥':15,\n",
    "    'يد عاشره':10,\n",
    "    '10':10,\n",
    "    '12':12,\n",
    "    'سابعه':7,\n",
    "    '14':14,\n",
    "    'يد ١٧':17,\n",
    "    'يد سابعة':7,\n",
    "    'ايد رابعه':4,\n",
    "    'يد ثالث':3,\n",
    "    'يد3':3,\n",
    "    'يد تاسعة':9,\n",
    "    'خامسه':5,\n",
    "    '٠٠٠٠٩':9,\n",
    "    '١٧':17,\n",
    "    '١٢':12,\n",
    "    '19':9,\n",
    "    '١٧':17,\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "  \n",
    " 'يد اولي 1' : 1\n",
    "\n",
    "}\n",
    "\n",
    "cars_data['ex_owners'] =  cars_data['ex_owners'].replace(value_mapping)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b570e5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe091b97",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b256aa",
   "metadata": {},
   "source": [
    "#  the feature contains noisy data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5bdc2a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop rows Which have noisy\n",
    "values_to_drop = ['12345', '1000', '؟', '0778244', 'E', '11111', '?', 'مشطوب','170000',  '@', '123423', 'اصل تدريب سواقه', 'اشخاص', 'شطب', '14','159000','نعم','منمره','78888877766','خصوصي','للبيع','56 ',]\n",
    "cars_data = cars_data[~cars_data['ex_owners'].isin(values_to_drop)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b019944",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "200ed386",
   "metadata": {},
   "source": [
    "# Filling ex_owners with most occurring Values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85861e3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filling with most occurring Values\n",
    "\n",
    "cars_data['ex_owners'].fillna(cars_data['ex_owners'].value_counts().index[0], inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8b36598",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1586041a",
   "metadata": {},
   "source": [
    "# convert The Type to int "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aef5c876",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'] = cars_data['ex_owners'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2de6f390",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f7d7110",
   "metadata": {},
   "outputs": [],
   "source": [
    "def box_plot(The_feature):\n",
    "    data_column = The_feature\n",
    "    fig, ax = plt.subplots(figsize=(20, 10))\n",
    "    ax.boxplot(data_column, vert=False)\n",
    "    ax.set_xlabel('Feature')\n",
    "    ax.set_title('Box Plot of Column1')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb439d01",
   "metadata": {},
   "outputs": [],
   "source": [
    "def hist_plot (The_feature) :\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.histplot(The_feature, kde=True, color='pink')\n",
    "    plt.title(f'Distribution of {The_feature}')\n",
    "    plt.xlabel(feature_to_plot)\n",
    "    plt.ylabel('Frequency')\n",
    "    skewness = dr.skew()\n",
    "    print(f\"Skewness: {skewness}\")\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f70173a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def identify_outliers_iqr(The_feature):\n",
    "    Q1 = The_feature.quantile(0.25)\n",
    "    Q3 = The_feature.quantile(0.75)\n",
    "    IQR = Q3 - Q1\n",
    "    lower_bound = Q1 - 1.5 * IQR\n",
    "    upper_bound = Q3 + 1.5 * IQR\n",
    "    return [  lower_bound , upper_bound]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd690d23",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Remove_the_outliers(data , The_feature , lowest_value , highest_value):\n",
    "    cleaned_data = data[(The_feature >= lowest_value) & (The_feature <= highest_value)]\n",
    "    return cleaned_data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "249042d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['ex_owners'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ab9fbcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "identify_outliers_iqr(cars_data['ex_owners'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75732489",
   "metadata": {},
   "source": [
    "# Most cars have only one previous owner, while cars that have more than 5 are the minority"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e17c75f",
   "metadata": {},
   "source": [
    "# Remove the outliers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f3bd149",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data =  Remove_the_outliers(cars_data ,cars_data['ex_owners'] , 0 , 5 ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1993fd96",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['ex_owners'].value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb346c04",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['ex_owners'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79253ee2",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d0f6fee",
   "metadata": {},
   "source": [
    "# motor_power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14a36f89",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['motor_power'].dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01b0e31a",
   "metadata": {},
   "source": [
    "# motor power data type is float64"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48609e08",
   "metadata": {},
   "source": [
    "# Covert the data type to int "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c6e22c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['motor_power'] = cars_data['motor_power'].astype('int64')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "578768b7",
   "metadata": {},
   "source": [
    "# identifty the outliers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68ba73f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "identify_outliers_iqr(cars_data['motor_power'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0acb3249",
   "metadata": {},
   "source": [
    "# the normal range for the 'قوة الماتور ' is between [500,2900] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d4347f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['motor_power'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "139f021e",
   "metadata": {},
   "source": [
    "# Remove the outliers from motor power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d9eb7fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = Remove_the_outliers(cars_data ,cars_data['motor_power'], 500 , 3000 ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b611f77c",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['motor_power'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7a9c197",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95031383",
   "metadata": {},
   "source": [
    "# car_speedometer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30ff0612",
   "metadata": {},
   "source": [
    "# There are numeric values, strings, and symbols in this feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fdef87d",
   "metadata": {},
   "outputs": [],
   "source": [
    " cars_data['car_speedometer'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8f8f22a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Trying to standardize the data type \n",
    "\n",
    "#delete the strings \n",
    "pattern = r'[أ-يا]'\n",
    "rows_with_letters = cars_data['car_speedometer'].str.contains(pattern, na=False, regex=True)\n",
    "num_rows_with_letters_before_drop = cars_data[rows_with_letters].shape[0]\n",
    "num_rows_with_letters_before_drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4ba8a42",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = cars_data[~rows_with_letters]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41a3747f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15fda8b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_copy = cars_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3958906",
   "metadata": {},
   "outputs": [],
   "source": [
    "arabic_to_english = {\n",
    "    '٠': '0',\n",
    "    '١': '1',\n",
    "    '٢': '2',\n",
    "    '٣': '3',\n",
    "    '٤': '4',\n",
    "    '٥': '5',\n",
    "    '٦': '6',\n",
    "    '٧': '7',\n",
    "    '٨': '8',\n",
    "    '٩': '9'\n",
    "}\n",
    "\n",
    "def convert_arabic_to_english(arabic_number):\n",
    "    if pd.notna(arabic_number):\n",
    "        for arabic, english in arabic_to_english.items():\n",
    "            arabic_number = arabic_number.replace(arabic, english)\n",
    "        \n",
    "        # Remove non-numeric characters\n",
    "        arabic_number = re.sub(r'[^0-9]', '0' , arabic_number)\n",
    "        print(arabic_number)\n",
    "\n",
    "    return arabic_number\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0a4eb57",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'] = cars_data['car_speedometer'].apply(convert_arabic_to_english)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8876289",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer']  = pd.to_numeric(cars_data['car_speedometer'] , errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57dd038c",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93454128",
   "metadata": {},
   "outputs": [],
   "source": [
    " cars_data['car_speedometer'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6baab3ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f47b39b",
   "metadata": {},
   "source": [
    "# there is a missing value in this feature . "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d65dc145",
   "metadata": {},
   "source": [
    "# Identify outliers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5e96379",
   "metadata": {},
   "outputs": [],
   "source": [
    "lowest , highest = identify_outliers_iqr(cars_data['car_speedometer'])\n",
    "print(lowest , highest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e539ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_outliers = cars_data[(cars_data['car_speedometer']  < lowest) | (cars_data['car_speedometer'] > highest)]\n",
    "len(number_of_outliers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a1b0b4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test = cars_data.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d4a197e",
   "metadata": {},
   "source": [
    "# Fill the missing  with median "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbfcf29d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'].fillna(cars_data['car_speedometer'].median(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4502255f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Q1 = cars_data['car_speedometer'].quantile(0.25)\n",
    "Q3 = cars_data['car_speedometer'].quantile(0.85)\n",
    "IQR = Q3 - Q1\n",
    "lower_bound_in_speedometer = Q1 - 1.5 * IQR\n",
    "upper_bound_in_speedometer = Q3 + 1.5 * IQR\n",
    "lower_bound_in_speedometer , upper_bound_in_speedometer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e79549e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = cars_data[(cars_data['car_speedometer'] > lower_bound_in_speedometer) & (cars_data['car_speedometer'] < upper_bound_in_speedometer ) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16a85e86",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['car_speedometer'] .max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbba08b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['car_speedometer'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a42ed094",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "722efecf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = cars_data.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2e5c90f",
   "metadata": {},
   "source": [
    "# Now the range of value in the car_speedometer is between [0,333000]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a013942a",
   "metadata": {},
   "source": [
    "# price\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70e64baf",
   "metadata": {},
   "source": [
    "# The data type is object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e987ca2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2164b998",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cars_data['Price'] = cars_data ['Price'].astype(str).str.replace('شيكل', '')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90f390eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.rename(columns={'Price' : 'Price(NIS)'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fe238df",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'] .isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1bba84e",
   "metadata": {},
   "source": [
    "# There is no missing value in the price "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcd8bef3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'] = cars_data['Price(NIS)'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c043b4dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841a1792",
   "metadata": {},
   "source": [
    "# Identify the outliers in Price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10120f02",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['Price(NIS)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16fef429",
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_bound, upper_bound   = identify_outliers_iqr(cars_data['Price(NIS)'])\n",
    "lower_bound, upper_bound "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe9f2a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_outliers = ((cars_data['Price(NIS)'] < lower_bound) | (cars_data['Price(NIS)'] > upper_bound)).sum()\n",
    "print(f\"Number of outliers in 'Price(NIS)': {num_outliers}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddfe5be5",
   "metadata": {},
   "outputs": [],
   "source": [
    " cars_data[cars_data['Price(NIS)'] < 5000][0:50]\n",
    "#its fine with this prices , the price sutible with modle and speedometer . "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0947b503",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    " cars_data[cars_data['Price(NIS)'] > upper_bound ][0:50]\n",
    "#Unsuitable or unnatural prices. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5110e17e",
   "metadata": {},
   "source": [
    "# Remove the outliers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f72dfe05",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = Remove_the_outliers(cars_data , cars_data['Price(NIS)'] , 0 , 155000.0 ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "830f35bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['Price(NIS)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee7bc341",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b724d363",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d0727c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beb635ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81d76065",
   "metadata": {},
   "source": [
    "# Modle feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14a9084f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81c49a23",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['model'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4e5765f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['model'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc57897e",
   "metadata": {},
   "source": [
    "# There is No missing data "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11a412bc",
   "metadata": {},
   "source": [
    "# Convert the data type "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d56cc42c",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['model'] = cars_data ['model'].astype(str).str.replace('موديل سنة', '')\n",
    "cars_data['model'] = cars_data['model'].astype('int64')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce43f760",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['model'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e3862a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_values_in_the_features = cars_data.isnull().sum()\n",
    "missing_values_in_the_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3da8316e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filling with most occurring Values\n",
    "#cars_data = cars_data.apply(lambda x: x.fillna(x.value_counts().index[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6be588f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['model'].value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "093e8b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(cars_data['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b6bea7b",
   "metadata": {},
   "outputs": [],
   "source": [
    " cars_data[cars_data['model'] < 1995][0:50]\n",
    "#this data can be consider normal data not outliers "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a887fa2",
   "metadata": {},
   "source": [
    "# passengers Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4948aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a7872ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fc1cc87",
   "metadata": {},
   "source": [
    "# There is missing value in this feature "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffb95ff0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "278378e7",
   "metadata": {},
   "source": [
    "# mapping to standrize the format  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cbc2916",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].unique()\n",
    "noisy_data_in_passengers = []\n",
    "\n",
    "mapping_for_passengers  =  { \n",
    "  '٤+١' : '4+1' ,\n",
    "  '4+١' : '4+1' , \n",
    "  '١+٤' : '4+1' , \n",
    "  '1+٤' : '4+1' , \n",
    "  '4\\\\١': '4+1', \n",
    "  '7,+ 1': '7+1',\n",
    "  '7:1' : '7+1' ,\n",
    "  '7.1' : '7+1' , \n",
    "  '٧+1' :  '7+1' , \n",
    "  '1+7' : '7+1' , \n",
    "  '7 =1': '7+1' , \n",
    "  '٧+١' :  '7+1',  \n",
    "  '6+١' : '6+1' ,\n",
    "  '6x1' : '6+1' , \n",
    "  '٦+١': '6+1' , \n",
    "  '٨+١' : '8+1' ,\n",
    "  'nan' : 'NaN',\n",
    "  '6±١': '6+1' , \n",
    "  '٥+١'  : '5+1' , \n",
    "  '4+2' : '5+1' ,\n",
    "} \n",
    "cars_data['passengers'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "577ddd3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'] =  cars_data['passengers'].replace(mapping_for_passengers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b807910",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be3ea57c",
   "metadata": {},
   "outputs": [],
   "source": [
    "The_noisy_in_passengers = [ '4+2', '3+1', '1', 'اكثر من 10', '9+1', '1+1' , '٧' ,  '+1', '1+2', '2/1' , '2' , '22']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "306d19f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data = cars_data[~cars_data['passengers'].isin(The_noisy_in_passengers)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "067692a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c3da6cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41cdcf61",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = cars_data.copy()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb446bf6",
   "metadata": {},
   "source": [
    "# Filling the missing data in passengers using Knn algorithem "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29185bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def impute_non_numeric(neighbors):\n",
    "    # Extract the 'passengers' values from the neighbors\n",
    "    passengers_values = neighbors['passengers'].dropna()\n",
    "    print( 'Hello' , passengers_values)\n",
    "    # Use mode to find the most common category\n",
    "    if not passengers_values.empty:\n",
    "        return mode(passengers_values)\n",
    "#else all the neighbors , there passengers_values is missing .\n",
    "\n",
    "   \n",
    "numeric_features = df[['model', 'Price(NIS)', 'motor_power', 'car_speedometer']]\n",
    "non_numeric_feature = 'passengers'\n",
    "\n",
    "missing_indices = df[df[non_numeric_feature].isnull()].index\n",
    "knn_model = NearestNeighbors(n_neighbors = 5)\n",
    "knn_model.fit(numeric_features)\n",
    "\n",
    "for index in missing_indices:\n",
    "    print(index)\n",
    "    features_to_impute = numeric_features.loc[index].values.reshape(1, -1)\n",
    "    print(features_to_impute)\n",
    "    distances, indices = knn_model.kneighbors(features_to_impute)\n",
    "    print(distances, indices)\n",
    "    neighbors = df.iloc[indices[0]]\n",
    "    print(neighbors['passengers'])\n",
    "    df.at[index, non_numeric_feature] = impute_non_numeric(neighbors)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dbff691",
   "metadata": {},
   "outputs": [],
   "source": [
    "# when k = 1 => 594\n",
    "# k => 2 =>110\n",
    "#k => 3 => 28\n",
    "#k => 5 =>3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d1afe70",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['passengers'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b372385a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'] = df['passengers']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d8811a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data= cars_data.dropna(subset=['passengers'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d12d126",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe4a5b49",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(cars_data['passengers'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "414940e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d2ca56",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def pie_plot(The_feature):\n",
    "    feature_counts = The_feature.value_counts()\n",
    "    colors = plt.cm.Paired(range(len(feature_counts)))\n",
    "    plt.figure(figsize=(20, 10))\n",
    "    patches, texts, autotexts = plt.pie(feature_counts, labels=feature_counts.index, autopct='',\n",
    "                                       startangle=140, colors=colors)\n",
    "    for text, autotext, color in zip(texts, autotexts, colors):\n",
    "        text.set_color(color)\n",
    "        autotext.set_color(color)\n",
    "\n",
    "    plt.title('Pie Chart for Feature')\n",
    "    plt.rcParams['font.weight'] = 'bold'\n",
    "    plt.rcParams['font.size'] = 10\n",
    "\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451f2628",
   "metadata": {},
   "outputs": [],
   "source": [
    "pie_plot(cars_data['passengers'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "318ec717",
   "metadata": {},
   "source": [
    "# convert the passengers to int  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd971937",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4a7563f",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9d0f48a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_numeric(value):\n",
    "    if isinstance(value, str):\n",
    "        parts = value.split('+')\n",
    "        if len(parts) == 2:\n",
    "            return int(parts[0]) + int(parts[1])\n",
    "    return value  # Return the original value for non-string or invalid formats\n",
    "\n",
    "cars_data['passengers'] = cars_data['passengers'].apply(convert_to_numeric)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7b3a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06e2b856",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'] = cars_data['passengers'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1ddbdeb",
   "metadata": {},
   "outputs": [],
   "source": [
    "missing_values_in_the_features = cars_data.isnull().sum()\n",
    "missing_values_in_the_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09f8bfd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Number_of_Additions'] = cars_data['Number_of_Additions'].astype('int64')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8a5a94b",
   "metadata": {},
   "source": [
    "# Now all features (numeric Features) having no missing values OR outliers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52d55577",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "031c838b",
   "metadata": {},
   "source": [
    "# Try to normalize and Handle Skewed Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e01e0fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from_here = cars_data.copy()\n",
    "from_here \n",
    "cars_data = from_here.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd62dfa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.kurt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f62430ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.skew()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "388ad4f1",
   "metadata": {},
   "source": [
    "# Normlaize model  : -1.452954  ,  passengers :  2.147018 , Price(NIS) : 0.727353 "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82a30942",
   "metadata": {},
   "source": [
    "# Most of Our data is positively skewed => right tail of the distribution \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee29820d",
   "metadata": {},
   "source": [
    "# But execpt modle and Number_of_Additions  is Negtivaly skewed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13bb2ee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_plot = ['Price(NIS)', 'motor_power', 'car_speedometer', 'passengers', 'ex_owners', 'Number_of_Additions', 'sunroof','model']\n",
    "\n",
    "fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(15, 8))\n",
    "fig.suptitle('Distributions of Features')\n",
    "\n",
    "axes = axes.flatten()\n",
    "\n",
    "for i, feature in enumerate(features_to_plot):\n",
    "    sns.histplot(cars_data[feature], kde=True,  color='Purple' ,  ax=axes[i])\n",
    "    axes[i].set_title(feature)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13d475bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def hist_plot (The_feature) : \n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.histplot(The_feature, kde=True, color='pink')\n",
    "    plt.xlabel('price_transformation')\n",
    "    plt.ylabel('Frequency')\n",
    "    skewness = The_feature.skew()\n",
    "    print(f\"Skewness: {skewness}\")\n",
    "    plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5456ed1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "price_log_transformation = log_transformation(cars_data['Price(NIS)'])\n",
    "hist_plot(price_log_transformation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6c251a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "price_Reciprocal_transformation = Reciprocal_transformation(cars_data['Price(NIS)'])\n",
    "hist_plot(price_Reciprocal_transformation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "318ddefc",
   "metadata": {},
   "outputs": [],
   "source": [
    "price_square_root_transformation = square_root_transformation(cars_data['Price(NIS)'])\n",
    "hist_plot(price_square_root_transformation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dab8a3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_to_plot = price_square_root_transformation\n",
    "sm.qqplot(feature_to_plot ,  line='s')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88f98dbd",
   "metadata": {},
   "source": [
    "# So we choose the square_root transformation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21502c65",
   "metadata": {},
   "outputs": [],
   "source": [
    "root_transformed_data = square_root_transformation(cars_data['Price(NIS)'])\n",
    "cars_data['Price(NIS)'] = root_transformed_data\n",
    "transformation_info = {'transformation_function': square_root_transformation, 'transformed_features': root_transformed_data}\n",
    "with open('root_transformation_info.pkl', 'wb') as file:\n",
    "    pickle.dump(transformation_info, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f86250ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb830b92",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a15adf5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "Reciprocal_transformed_data = Reciprocal_transformation(cars_data ['passengers'])\n",
    "cars_data ['passengers'] = Reciprocal_transformed_data\n",
    "Reciprocal_transformation_info = {'transformation_function': Reciprocal_transformation , 'transformed_features': Reciprocal_transformed_data}\n",
    "with open('Reciprocal_transformation_info.pkl', 'wb') as file:\n",
    "    pickle.dump(Reciprocal_transformation_info, file)\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18c3cdf7",
   "metadata": {},
   "source": [
    "# The Target Variable is approximately normal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2620e3e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Try to normlize the passengers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e6ba56",
   "metadata": {},
   "outputs": [],
   "source": [
    "passengers_log_transformation = log_transformation(cars_data['passengers'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fe12ce1",
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_plot(passengers_log_transformation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13f1c1b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "passengers_square_root = square_root_transformation(cars_data['passengers'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9c21869",
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_plot(passengers_square_root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5ce4d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "passengers_Reciprocal_transformation =  Reciprocal_transformation(cars_data['passengers'])\n",
    "hist_plot(passengers_Reciprocal_transformation)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5baff8e8",
   "metadata": {},
   "source": [
    "# tranforme data using Reciprocal_transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "814f569a",
   "metadata": {},
   "source": [
    "# value of Kurtosis of passengers : 5.342821 before transformation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f42477a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_reciprocal_transformation_info(dataframe, feature_name):\n",
    "    transformed_data = Reciprocal_transformation(dataframe[feature_name])\n",
    "    transformation_info = {'transformation_function': Reciprocal_transformation, 'transformed_features': transformed_data}\n",
    "    \n",
    "    with open(f'{feature_name}_transformation_info.pkl', 'wb') as file:\n",
    "        pickle.dump(transformation_info, file)\n",
    "\n",
    "        \n",
    "Reciprocal_transformed_data = Reciprocal_transformation(cars_data['passengers'])\n",
    "cars_data['passengers'] = Reciprocal_transformed_data\n",
    "save_reciprocal_transformation_info(cars_data, 'passengers')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de141ee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_plot(cars_data['passengers'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "895b32cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['passengers'].kurt() #WOW :) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "467fa62d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.skew()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "117ca773",
   "metadata": {},
   "outputs": [],
   "source": [
    "modle_square_root_transformation = square_root_transformation(cars_data['model'])\n",
    "hist_plot(modle_square_root_transformation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03aa2b02",
   "metadata": {},
   "outputs": [],
   "source": [
    "modle_Reciprocal_transformation =  Reciprocal_transformation(cars_data['model'])\n",
    "hist_plot(modle_Reciprocal_transformation )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "370adf08",
   "metadata": {},
   "outputs": [],
   "source": [
    "modle_log_transformation = log_transformation(cars_data['model'])\n",
    "hist_plot(modle_log_transformation)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e087fff8",
   "metadata": {},
   "source": [
    "# NO effect OF Transformation  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "870eb7b6",
   "metadata": {},
   "source": [
    "# Done of NORMLIZE Features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04653bbb",
   "metadata": {},
   "source": [
    "# Scalling - The numeric data between [0,1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b659e4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06741254",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_of_interest = ['Price(NIS)', 'model']\n",
    "\n",
    "# Create a pair plot\n",
    "sns.pairplot(cars_data[columns_of_interest], height=2.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a47296c",
   "metadata": {},
   "outputs": [],
   "source": [
    "after_normalized = min_max_normalization(cars_data , ['Price(NIS)', 'model'] )\n",
    "after_normalized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c2d47be",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_of_interest = ['Price(NIS)', 'model']\n",
    "\n",
    "# Create a pair plot\n",
    "sns.pairplot(after_normalized[columns_of_interest], height=2.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79c113d4",
   "metadata": {},
   "source": [
    "# This means that the Type of correlation after Scalling was not affected  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05c53772",
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_max_normalization(dataframe, features_to_normalize):\n",
    "    if not features_to_normalize:\n",
    "        # Handle case when features_to_normalize is empty\n",
    "        return dataframe\n",
    "\n",
    "    # Separate the 'Price(NIS)' feature from the rest\n",
    "    price_feature = 'Price(NIS)'\n",
    "    other_features = [feature for feature in features_to_normalize if feature != price_feature]\n",
    "\n",
    "    # Create a scaler for 'Price(NIS)'\n",
    "    scaler_price = MinMaxScaler()\n",
    "    dataframe[price_feature] = scaler_price.fit_transform(dataframe[price_feature].values.reshape(-1, 1))\n",
    "\n",
    "    # Create a scaler for the rest of the features\n",
    "    scaler_other = MinMaxScaler()\n",
    "    dataframe[other_features] = scaler_other.fit_transform(dataframe[other_features])\n",
    "\n",
    "    # Save the scalers and features to separate pickle files\n",
    "    with open('scaler_price.pkl', 'wb') as scaler_file_price:\n",
    "        pickle.dump({'scaler': scaler_price, 'features': [price_feature]}, scaler_file_price)\n",
    "\n",
    "    with open('scaler_other.pkl', 'wb') as scaler_file_other:\n",
    "        pickle.dump({'scaler': scaler_other, 'features': other_features}, scaler_file_other)\n",
    "\n",
    "    # Save information about the inverse transformation\n",
    "    inverse_transform_info_price = {'transformation_function': scaler_price.inverse_transform, 'features': [price_feature]}\n",
    "    inverse_transform_info_other = {'transformation_function': scaler_other.inverse_transform, 'features': other_features}\n",
    "\n",
    "    with open('inverse_transform_info_price.pkl', 'wb') as inverse_transform_file_price:\n",
    "        pickle.dump(inverse_transform_info_price, inverse_transform_file_price)\n",
    "\n",
    "    with open('inverse_transform_info_other.pkl', 'wb') as inverse_transform_file_other:\n",
    "        pickle.dump(inverse_transform_info_other, inverse_transform_file_other)\n",
    "\n",
    "    return dataframe\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3963b05",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "variables_columns_to_scall = ['Price(NIS)', 'model','motor_power' , 'car_speedometer' , 'passengers' , 'ex_owners' , 'Number_of_Additions' ]\n",
    "min_max_normalization(cars_data , variables_columns_to_scall )\n",
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f7105b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ac815e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d6831dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['Price(NIS)'].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98a11733",
   "metadata": {},
   "source": [
    "# Now compute The corelation between all numeric features and Target Variable "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80d4fb81",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_plot = ['Price(NIS)', 'motor_power', 'car_speedometer', 'passengers', 'ex_owners', 'Number_of_Additions', 'sunroof' ,'model']\n",
    "\n",
    "# Create subplots\n",
    "fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(15, 8))\n",
    "fig.suptitle('Scatter Plots and Correlations with Price')\n",
    "\n",
    "# Flatten the axes for easier indexing\n",
    "axes = axes.flatten()\n",
    "\n",
    "for i, feature in enumerate(features_to_plot):\n",
    "    sns.scatterplot(x=cars_data[feature], y=cars_data['Price(NIS)'], ax=axes[i])\n",
    "    correlation_coefficient = cars_data[feature].corr(cars_data['Price(NIS)'])\n",
    "    axes[i].annotate(f'Corr: {correlation_coefficient:.2f}', xy=(0.5, 1.05), xycoords='axes fraction', ha='center', fontsize=10)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1f07a5f",
   "metadata": {},
   "source": [
    "# The strongest relationship is between price and model, which is a positive relationship"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d7411d",
   "metadata": {},
   "source": [
    "# The weakest relationship is between price and the car’s speedometer, and their relationship is negative"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1ad72ce",
   "metadata": {},
   "source": [
    "# Categorical Feature "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0b68506",
   "metadata": {},
   "source": [
    "# glass feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a857c6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "font_properties = {'font.family': 'Arial', 'font.size': 10, 'font.sans-serif': ['Arial'], 'text.usetex': False}\n",
    "matplotlib.rcParams.update(font_properties)\n",
    "\n",
    "features_to_plot = ['fuel_type', 'car_license', 'lime_type', 'glass']\n",
    "\n",
    "# Create subplots\n",
    "fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(12, 10))\n",
    "fig.suptitle('Pie Charts of Categorical Features')\n",
    "\n",
    "# Flatten the axes for easier indexing\n",
    "axes = axes.flatten()\n",
    "\n",
    "for i, feature in enumerate(features_to_plot):\n",
    "    # Count the occurrences of each category in the feature\n",
    "    feature_counts = cars_data[feature].value_counts()\n",
    "\n",
    "    # Reshape and reorder Arabic text\n",
    "    labels = [get_display(arabic_reshaper.reshape(label)) for label in feature_counts.index]\n",
    "\n",
    "    # Plot a pie chart\n",
    "    pie = axes[i].pie(feature_counts, labels=labels, autopct='%1.1f%%', startangle=90)\n",
    "    axes[i].set_title(feature)\n",
    "\n",
    "plt.tight_layout(rect=[0, 0, 1, 0.96])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39fc623b",
   "metadata": {},
   "source": [
    "# This   features are imbalanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc246074",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "597b2669",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['origin_car'].unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba572589",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data['color'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63788c8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features = ['color', 'origin_car']\n",
    "fig, axes = plt.subplots(nrows=len(categorical_features), figsize=(10, 6 * len(categorical_features)))\n",
    "fig.suptitle('Bar Charts of Categorical Features', y=1.02)\n",
    "for i, feature in enumerate(categorical_features):\n",
    "    feature_counts = cars_data[feature].value_counts()\n",
    "    labels = [get_display(arabic_reshaper.reshape(label)) for label in feature_counts.index]\n",
    "    sns.countplot(x=feature, data=cars_data, ax=axes[i], order=feature_counts.index)\n",
    "    axes[i].set_title(f'Distribution of {feature}')\n",
    "    axes[i].set_xticklabels(labels)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93db1a76",
   "metadata": {},
   "source": [
    "# Convert the  features from nominal to Numeric "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72929617",
   "metadata": {},
   "source": [
    "# the features that have two - values use LabelEncoder to convert it "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad671696",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Label_Encoder(The_feature):\n",
    "    label_encoder = LabelEncoder()\n",
    "    The_feature_encoded = label_encoder.fit_transform(The_feature)\n",
    "    return The_feature_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4dc2142",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_that_have_two_values = ['car_license', 'glass']\n",
    "for feature in features_that_have_two_values:\n",
    "    cars_data[feature] = Label_Encoder(cars_data[feature])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "988a315d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07c7af66",
   "metadata": {},
   "source": [
    "# Now use one-hot encoding to convert feature which contains more than two values (classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "059badbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def OneHot_Encoder(data, column):\n",
    "    onehot_encoded_columns = pd.get_dummies(data[column], prefix=f'{column}-')\n",
    "    data = pd.concat([data, onehot_encoded_columns], axis=1)\n",
    "    data.drop(column, axis=1, inplace=True)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b46d8912",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_have_more_than_two_values = ['Name','color', 'fuel_type','origin_car','lime_type']\n",
    "for feature in features_have_more_than_two_values:\n",
    "    cars_data= OneHot_Encoder(cars_data ,feature)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e210f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "cars_data.reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f75a5c34",
   "metadata": {},
   "source": [
    "\n",
    "# Now the data is ready to use "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5c54a86",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = cars_data.drop(['Price(NIS)'],axis=1)  \n",
    "y = cars_data['Price(NIS)']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf0fb3bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8eabbb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    'Linear Regression (Degree 1)': LinearRegression(),\n",
    "    'Ridge Regression (Degree 1)': Ridge(alpha=1.0),\n",
    "    'Lasso Regression (Degree 1)': Lasso(alpha=1.0),\n",
    "    'Polynomial Regression (Degree 2)': make_pipeline(PolynomialFeatures(degree=2), LinearRegression()),\n",
    "    'Ridge Regression (Degree 2)': make_pipeline(PolynomialFeatures(degree=2), Ridge(alpha=1.0)),\n",
    "    'Lasso Regression (Degree 2)': make_pipeline(PolynomialFeatures(degree=2), Lasso(alpha=1.0)),\n",
    "    'K-Nearest Neighbors': KNeighborsRegressor(n_neighbors=3),\n",
    "    'Decision Tree': DecisionTreeRegressor(random_state=0)\n",
    "}\n",
    "\n",
    "results = {}\n",
    "\n",
    "for model_name, model in models.items():\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred_train = model.predict(X_train)\n",
    "    y_pred_test = model.predict(X_test)\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    y_pred_train = model.predict(X_train)\n",
    "    y_pred_test = model.predict(X_test)\n",
    "    \n",
    "    mse_train = mean_squared_error(y_train, y_pred_train)\n",
    "    r2_train = r2_score(y_train, y_pred_train)\n",
    "    \n",
    "    mse_test = mean_squared_error(y_test, y_pred_test)\n",
    "    r2_test = r2_score(y_test, y_pred_test)\n",
    "    \n",
    "    results[model_name] = {\n",
    "        'MSE_train': mse_train,\n",
    "        'R2_train': r2_train,\n",
    "        'MSE_test': mse_test,\n",
    "        'R2_test': r2_test\n",
    "    }\n",
    "\n",
    "for model_name, metrics in results.items():\n",
    "    print(f\"{model_name}:\")\n",
    "    print(f\"R-squared (Training): {metrics['R2_train']}\")\n",
    "    print(f\"R-squared (Testing): {metrics['R2_test']}\")\n",
    "    \n",
    "    print(f\"Mean Squared Error (Training): {metrics['MSE_train']}\")\n",
    "    print(f\"Mean Squared Error (Testing): {metrics['MSE_test']}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f87d1780",
   "metadata": {},
   "source": [
    "# The Best Model for use is Ridge Regression (Degree 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d304fd1",
   "metadata": {},
   "source": [
    "# R-squerd for it 0.8 and MSE = 0.0077"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8ad9059",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming you have the results available in variables like r2_train, r2_test, mse_train, mse_test\n",
    "results_dict = {\n",
    "    'Linear Regression (Degree 1)': {\n",
    "        'R-squared (Training)': 0.8590414261282553,\n",
    "        'R-squared (Testing)': -4.377152074964761e+19,\n",
    "        'Mean Squared Error (Training)': 0.004817314454157538,\n",
    "        'Mean Squared Error (Testing)': 1.635786825066841e+18,\n",
    "    },\n",
    "    'Ridge Regression (Degree 1)': {\n",
    "        'R-squared (Training)': 0.8456910183602393,\n",
    "        'R-squared (Testing)': 0.7928622692858215,\n",
    "        'Mean Squared Error (Training)': 0.005273569866959013,\n",
    "        'Mean Squared Error (Testing)': 0.007740950395908376,\n",
    "    },\n",
    "    'Lasso Regression (Degree 1)': {\n",
    "        'R-squared (Training)': 0.0,\n",
    "        'R-squared (Testing)': -0.0004996010914921634,\n",
    "        'Mean Squared Error (Training)': 0.0341753915483049,\n",
    "        'Mean Squared Error (Testing)': 0.037389700835634525,\n",
    "    },\n",
    "    'Polynomial Regression (Degree 2)': {\n",
    "        'R-squared (Training)': 0.9594533955779442,\n",
    "        'R-squared (Testing)': -7.196364751207202e+21,\n",
    "        'Mean Squared Error (Training)': 0.0013856960820779875,\n",
    "        'Mean Squared Error (Testing)': 2.689355646501025e+20,\n",
    "    },\n",
    "    'Ridge Regression (Degree 2)': {\n",
    "        'R-squared (Training)': 0.9401354600084536,\n",
    "        'R-squared (Testing)': 0.7974223935561945,\n",
    "        'Mean Squared Error (Training)': 0.002045894094070254,\n",
    "        'Mean Squared Error (Testing)': 0.007570533853956182,\n",
    "    },\n",
    "    'Lasso Regression (Degree 2)': {\n",
    "        'R-squared (Training)': 0.0,\n",
    "        'R-squared (Testing)': -0.0004996010914921634,\n",
    "        'Mean Squared Error (Training)': 0.0341753915483049,\n",
    "        'Mean Squared Error (Testing)': 0.037389700835634525,\n",
    "    },\n",
    "    'K-Nearest Neighbors': {\n",
    "        'R-squared (Training)': 0.8119308307004252,\n",
    "        'R-squared (Testing)': 0.6008887607152252,\n",
    "        'Mean Squared Error (Training)': 0.006427337498977414,\n",
    "        'Mean Squared Error (Testing)': 0.014915198187702674,\n",
    "    },\n",
    "    'Decision Tree': {\n",
    "        'R-squared (Training)': 0.999869942636763,\n",
    "        'R-squared (Testing)': 0.6482851582333176,\n",
    "        'Mean Squared Error (Training)': 4.444761312365821e-06,\n",
    "        'Mean Squared Error (Testing)': 0.013143945983348991,\n",
    "    },\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "debe378d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TO print the result in clear way"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3291089b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for model_name, metrics in results_dict.items():\n",
    "    print(model_name + \":\")\n",
    "    for metric_name, value in metrics.items():\n",
    "        print(f\"{metric_name}: {value}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c156d362",
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge_model = Ridge(alpha=1.0)\n",
    "ridge_model.fit(X_train, y_train)\n",
    "y_pred_train = ridge_model.predict(X_train)\n",
    "y_pred_test = ridge_model.predict(X_test)\n",
    "\n",
    "mse_train = mean_squared_error(y_train, y_pred_train)\n",
    "r2_train = r2_score(y_train, y_pred_train)\n",
    "    \n",
    "mse_test = mean_squared_error(y_test, y_pred_test)\n",
    "r2_test = r2_score(y_test, y_pred_test)\n",
    "\n",
    "print(f\"R-squared (Training):\" ,r2_train )\n",
    "print(f\"R-squared (Testing): \" , r2_test)\n",
    "    \n",
    "print(f\"Mean Squared Error (Training):\" ,mse_train )\n",
    "print(f\"Mean Squared Error (Testing):\",mse_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c1055e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ridge_model.pkl', 'wb') as model_file:\n",
    "    pickle.dump(ridge_model, model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9335447b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d994b567",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}