From b6a1a321f1aa2f252749d701a4aa3c78ef1d618c Mon Sep 17 00:00:00 2001 From: rahulch-1 Date: Wed, 25 Oct 2023 14:42:02 +0530 Subject: [PATCH 1/4] Added Language dettector --- Language_Detector/Language_Detector.ipynb | 201 ++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 Language_Detector/Language_Detector.ipynb diff --git a/Language_Detector/Language_Detector.ipynb b/Language_Detector/Language_Detector.ipynb new file mode 100644 index 0000000..d1c9015 --- /dev/null +++ b/Language_Detector/Language_Detector.ipynb @@ -0,0 +1,201 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "JFA36TEH-AE-", + "outputId": "89a4bc51-46b8-412d-bdf1-ffef3d589307" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + " Text language\n", + "0 klement gottwaldi surnukeha palsameeriti ning ... Estonian\n", + "1 sebes joseph pereira thomas på eng the jesuit... Swedish\n", + "2 ถนนเจริญกรุง อักษรโรมัน thanon charoen krung เ... Thai\n", + "3 விசாகப்பட்டினம் தமிழ்ச்சங்கத்தை இந்துப் பத்திர... Tamil\n", + "4 de spons behoort tot het geslacht haliclona en... Dutch\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.naive_bayes import MultinomialNB\n", + "data = pd.read_csv(\"https://raw.githubusercontent.com/amankharwal/Website-data/master/dataset.csv\")\n", + "print(data.head())" + ] + }, + { + "cell_type": "code", + "source": [ + "data.isnull().sum()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fMGDxo3C-F6R", + "outputId": "40f8d248-5edc-41a0-d9b7-b6f557e87159" + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Text 0\n", + "language 0\n", + "dtype: int64" + ] + }, + "metadata": {}, + "execution_count": 2 + } + ] + }, + { + "cell_type": "code", + "source": [ + "data[\"language\"].value_counts()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-gNHLZik-Iac", + "outputId": "b610388f-bffa-4200-c6e7-9b8dd0fdf554" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Estonian 1000\n", + "Swedish 1000\n", + "English 1000\n", + "Russian 1000\n", + "Romanian 1000\n", + "Persian 1000\n", + "Pushto 1000\n", + "Spanish 1000\n", + "Hindi 1000\n", + "Korean 1000\n", + "Chinese 1000\n", + "French 1000\n", + "Portugese 1000\n", + "Indonesian 1000\n", + "Urdu 1000\n", + "Latin 1000\n", + "Turkish 1000\n", + "Japanese 1000\n", + "Dutch 1000\n", + "Tamil 1000\n", + "Thai 1000\n", + "Arabic 1000\n", + "Name: language, dtype: int64" + ] + }, + "metadata": {}, + "execution_count": 3 + } + ] + }, + { + "cell_type": "code", + "source": [ + "x = np.array(data[\"Text\"])\n", + "y = np.array(data[\"language\"])\n", + "\n", + "cv = CountVectorizer()\n", + "X = cv.fit_transform(x)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y,\n", + " test_size=0.33,\n", + " random_state=42)" + ], + "metadata": { + "id": "sfOgvFr1-JJU" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "model = MultinomialNB()\n", + "model.fit(X_train,y_train)\n", + "model.score(X_test,y_test)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ij2j-t5--LOQ", + "outputId": "a4590985-d4d9-41db-909f-23b58d4d3567" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.953168044077135" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ] + }, + { + "cell_type": "code", + "source": [ + "user = input(\"Enter a Text: \")\n", + "data = cv.transform([user]).toarray()\n", + "output = model.predict(data)\n", + "print(output)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "hu2_vXk4-Ndu", + "outputId": "5c49d5b7-2452-49a9-9843-d88059ea0af9" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Enter a Text: Bonjour, je m'appelle Rahul\n", + "['French']\n" + ] + } + ] + } + ] +} \ No newline at end of file From 2c3004430e79c596761d553abfd868f6c209d6c4 Mon Sep 17 00:00:00 2001 From: rahulch-1 Date: Wed, 25 Oct 2023 14:45:57 +0530 Subject: [PATCH 2/4] Readme file added --- Language_Detector/Readme.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 Language_Detector/Readme.md diff --git a/Language_Detector/Readme.md b/Language_Detector/Readme.md new file mode 100644 index 0000000..c9071ea --- /dev/null +++ b/Language_Detector/Readme.md @@ -0,0 +1 @@ +This is a Language Detector which Detects the language which is entered as an input when you run the program. \ No newline at end of file From 414c13fbac7d28effb454b7d609a8f3bac90b303 Mon Sep 17 00:00:00 2001 From: rahulch-1 <105146716+rahulch-1@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:41:56 +0530 Subject: [PATCH 3/4] Update Readme.md --- Language_Detector/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Language_Detector/Readme.md b/Language_Detector/Readme.md index c9071ea..58dd00d 100644 --- a/Language_Detector/Readme.md +++ b/Language_Detector/Readme.md @@ -1 +1 @@ -This is a Language Detector which Detects the language which is entered as an input when you run the program. \ No newline at end of file +Language detection is a natural language processing task where we need to identify the language of a text or document. Using machine learning for language identification was a difficult task a few years ago because there was not a lot of data on languages, but with the availability of data with ease, several powerful machine learning models are already available for language identification. So, if you want to learn how to train a machine learning model for language detection, then this is for you From 79d873df9609041b21f23828f646587ca7a36a45 Mon Sep 17 00:00:00 2001 From: rahulch-1 <105146716+rahulch-1@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:42:21 +0530 Subject: [PATCH 4/4] Update Readme.md --- Language_Detector/Readme.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Language_Detector/Readme.md b/Language_Detector/Readme.md index 58dd00d..47f22dd 100644 --- a/Language_Detector/Readme.md +++ b/Language_Detector/Readme.md @@ -1 +1,3 @@ +# LANGUAGE DETECTOR USING PYTHON + Language detection is a natural language processing task where we need to identify the language of a text or document. Using machine learning for language identification was a difficult task a few years ago because there was not a lot of data on languages, but with the availability of data with ease, several powerful machine learning models are already available for language identification. So, if you want to learn how to train a machine learning model for language detection, then this is for you