RI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5d4bbba9-eaf9-4e36-84b7-36509d5ee820",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import math\n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import nltk\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tkinter as tk\n",
    "from tkinter import ttk, messagebox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dab7a981-e437-406c-aecf-c7c6a81f5783",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\LENOVO\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# nltk resources \n",
    "try:\n",
    "    nltk.data.find('tokenizers/punkt')\n",
    "except LookupError:\n",
    "    nltk.download('punkt')\n",
    "\n",
    "try:\n",
    "    nltk.data.find('corpora/stopwords')\n",
    "except LookupError:\n",
    "    nltk.download('stopwords')\n",
    "\n",
    "try:\n",
    "    nltk.data.find('corpora/wordnet')\n",
    "except LookupError:\n",
    "    nltk.download('wordnet')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9812b172-5329-4bfa-b361-a0832fb50e31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load corpus\n",
    "corpus_file = 'trec-covid/corpus.jsonl'\n",
    "cleaned_file = 'cleaned_corpus.jsonl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7fe2e8a2-c9b5-46da-90c9-b7bfcc5d61cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_id': 'ug7v899j', 'title': 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', 'text': 'OBJECTIVE: This retrospective chart review describes the epidemiology and clinical features of 40 patients with culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia. METHODS: Patients with positive M. pneumoniae cultures from respiratory specimens from January 1997 through December 1998 were identified through the Microbiology records. Charts of patients were reviewed. RESULTS: 40 patients were identified, 33 (82.5%) of whom required admission. Most infections (92.5%) were community-acquired. The infection affected all age groups but was most common in infants (32.5%) and pre-school children (22.5%). It occurred year-round but was most common in the fall (35%) and spring (30%). More than three-quarters of patients (77.5%) had comorbidities. Twenty-four isolates (60%) were associated with pneumonia, 14 (35%) with upper respiratory tract infections, and 2 (5%) with bronchiolitis. Cough (82.5%), fever (75%), and malaise (58.8%) were the most common symptoms, and crepitations (60%), and wheezes (40%) were the most common signs. Most patients with pneumonia had crepitations (79.2%) but only 25% had bronchial breathing. Immunocompromised patients were more likely than non-immunocompromised patients to present with pneumonia (8/9 versus 16/31, P = 0.05). Of the 24 patients with pneumonia, 14 (58.3%) had uneventful recovery, 4 (16.7%) recovered following some complications, 3 (12.5%) died because of M pneumoniae infection, and 3 (12.5%) died due to underlying comorbidities. The 3 patients who died of M pneumoniae pneumonia had other comorbidities. CONCLUSION: our results were similar to published data except for the finding that infections were more common in infants and preschool children and that the mortality rate of pneumonia in patients with comorbidities was high.', 'metadata': {'url': 'https://www.ncbi.nlm.nih.gov/pmc/articles/PMC35282/', 'pubmed_id': '11472636'}}\n",
      "{'_id': '02tnwd4m', 'title': 'Nitric oxide: a pro-inflammatory mediator in lung disease?', 'text': 'Inflammatory diseases of the respiratory tract are commonly associated with elevated production of nitric oxide (NO•) and increased indices of NO• -dependent oxidative stress. Although NO• is known to have anti-microbial, anti-inflammatory and anti-oxidant properties, various lines of evidence support the contribution of NO• to lung injury in several disease models. On the basis of biochemical evidence, it is often presumed that such NO• -dependent oxidations are due to the formation of the oxidant peroxynitrite, although alternative mechanisms involving the phagocyte-derived heme proteins myeloperoxidase and eosinophil peroxidase might be operative during conditions of inflammation. Because of the overwhelming literature on NO• generation and activities in the respiratory tract, it would be beyond the scope of this commentary to review this area comprehensively. Instead, it focuses on recent evidence and concepts of the presumed contribution of NO• to inflammatory diseases of the lung.', 'metadata': {'url': 'https://www.ncbi.nlm.nih.gov/pmc/articles/PMC59543/', 'pubmed_id': '11667967'}}\n"
     ]
    }
   ],
   "source": [
    "with open(corpus_file, 'r') as file:\n",
    "    documents = [json.loads(line) for line in file][:70]\n",
    "\n",
    "\n",
    "for doc in documents[:2]:\n",
    "    print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "be62a1a1-e6c2-4f8b-93d2-ff1f0467b388",
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean text function\n",
    "def clean_text(text):\n",
    "    text = re.sub(r'http\\S+', '', text)  # remove URLs\n",
    "    text = re.sub(r'[^a-zA-Z\\s]', '', text)  # remove non-alphabetic characters\n",
    "    text = text.lower()  # lowercase\n",
    "    tokens = word_tokenize(text)  # tokenize\n",
    "    tokens = [lemmatizer.lemmatize(word) for word in tokens if word not in stop_words]  # Lemmatize and remove stop words\n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "89f49182-e0d7-4563-b1f0-7d428c239bc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_docs = []\n",
    "for document in documents:\n",
    "    title = document.get('title', '')  \n",
    "    doc_id = document.get('_id', '') \n",
    "    doc_text = title + ' ' + document.get('text', '') \n",
    "    cleaned_tokens = clean_text(doc_text) \n",
    "    cleaned_docs.append({'id': doc_id, 'title': title, 'text': cleaned_tokens})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bce6ed15-244d-4cd9-bfc7-f164b3b047de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 'ug7v899j', 'title': 'Clinical features of culture-proven Mycoplasma pneumoniae infections at King Abdulaziz University Hospital, Jeddah, Saudi Arabia', 'text': 'clinical feature cultureproven mycoplasma pneumoniae infection king abdulaziz university hospital jeddah saudi arabia objective retrospective chart review describes epidemiology clinical feature patient cultureproven mycoplasma pneumoniae infection king abdulaziz university hospital jeddah saudi arabia method patient positive pneumoniae culture respiratory specimen january december identified microbiology record chart patient reviewed result patient identified required admission infection communityacquired infection affected age group common infant preschool child occurred yearround common fall spring threequarters patient comorbidities twentyfour isolates associated pneumonia upper respiratory tract infection bronchiolitis cough fever malaise common symptom crepitation wheeze common sign patient pneumonia crepitation bronchial breathing immunocompromised patient likely nonimmunocompromised patient present pneumonia versus p patient pneumonia uneventful recovery recovered following complication died pneumoniae infection died due underlying comorbidities patient died pneumoniae pneumonia comorbidities conclusion result similar published data except finding infection common infant preschool child mortality rate pneumonia patient comorbidities high'}\n",
      "{'id': '02tnwd4m', 'title': 'Nitric oxide: a pro-inflammatory mediator in lung disease?', 'text': 'nitric oxide proinflammatory mediator lung disease inflammatory disease respiratory tract commonly associated elevated production nitric oxide increased index dependent oxidative stress although known antimicrobial antiinflammatory antioxidant property various line evidence support contribution lung injury several disease model basis biochemical evidence often presumed dependent oxidation due formation oxidant peroxynitrite although alternative mechanism involving phagocytederived heme protein myeloperoxidase eosinophil peroxidase might operative condition inflammation overwhelming literature generation activity respiratory tract would beyond scope commentary review area comprehensively instead focus recent evidence concept presumed contribution inflammatory disease lung'}\n"
     ]
    }
   ],
   "source": [
    "for doc in cleaned_docs[:2]:\n",
    "    print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "36266908-4dc9-4549-bd89-97aca03ae608",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(cleaned_file, 'w') as file:\n",
    "    for doc in cleaned_docs:\n",
    "        file.write(json.dumps(doc) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c4d35610-c2bf-489c-9940-8ddaae6d1e19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate TF and IDF\n",
    "term_frequency = []\n",
    "document_frequency = Counter()\n",
    "\n",
    "for document in cleaned_docs:\n",
    "    tokens = document['text'].split()\n",
    "    term_counts = Counter(tokens)\n",
    "    total_terms = len(tokens)\n",
    "    tf = {term: count / total_terms for term, count in term_counts.items()}\n",
    "    term_frequency.append(tf)\n",
    "    document_frequency.update(set(tokens))\n",
    "\n",
    "unique_terms = list(document_frequency.keys())\n",
    "total_documents = len(cleaned_docs)\n",
    "idf = {term: math.log(total_documents / (1 + df)) for term, df in document_frequency.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cf7ea24c-d683-4d46-8cdf-33f15a2e628d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'clinical': 0.014285714285714285,\n",
       "  'feature': 0.014285714285714285,\n",
       "  'cultureproven': 0.014285714285714285,\n",
       "  'mycoplasma': 0.014285714285714285,\n",
       "  'pneumoniae': 0.03571428571428571,\n",
       "  'infection': 0.05,\n",
       "  'king': 0.014285714285714285,\n",
       "  'abdulaziz': 0.014285714285714285,\n",
       "  'university': 0.014285714285714285,\n",
       "  'hospital': 0.014285714285714285,\n",
       "  'jeddah': 0.014285714285714285,\n",
       "  'saudi': 0.014285714285714285,\n",
       "  'arabia': 0.014285714285714285,\n",
       "  'objective': 0.007142857142857143,\n",
       "  'retrospective': 0.007142857142857143,\n",
       "  'chart': 0.014285714285714285,\n",
       "  'review': 0.007142857142857143,\n",
       "  'describes': 0.007142857142857143,\n",
       "  'epidemiology': 0.007142857142857143,\n",
       "  'patient': 0.07857142857142857,\n",
       "  'method': 0.007142857142857143,\n",
       "  'positive': 0.007142857142857143,\n",
       "  'culture': 0.007142857142857143,\n",
       "  'respiratory': 0.014285714285714285,\n",
       "  'specimen': 0.007142857142857143,\n",
       "  'january': 0.007142857142857143,\n",
       "  'december': 0.007142857142857143,\n",
       "  'identified': 0.014285714285714285,\n",
       "  'microbiology': 0.007142857142857143,\n",
       "  'record': 0.007142857142857143,\n",
       "  'reviewed': 0.007142857142857143,\n",
       "  'result': 0.014285714285714285,\n",
       "  'required': 0.007142857142857143,\n",
       "  'admission': 0.007142857142857143,\n",
       "  'communityacquired': 0.007142857142857143,\n",
       "  'affected': 0.007142857142857143,\n",
       "  'age': 0.007142857142857143,\n",
       "  'group': 0.007142857142857143,\n",
       "  'common': 0.03571428571428571,\n",
       "  'infant': 0.014285714285714285,\n",
       "  'preschool': 0.014285714285714285,\n",
       "  'child': 0.014285714285714285,\n",
       "  'occurred': 0.007142857142857143,\n",
       "  'yearround': 0.007142857142857143,\n",
       "  'fall': 0.007142857142857143,\n",
       "  'spring': 0.007142857142857143,\n",
       "  'threequarters': 0.007142857142857143,\n",
       "  'comorbidities': 0.02857142857142857,\n",
       "  'twentyfour': 0.007142857142857143,\n",
       "  'isolates': 0.007142857142857143,\n",
       "  'associated': 0.007142857142857143,\n",
       "  'pneumonia': 0.04285714285714286,\n",
       "  'upper': 0.007142857142857143,\n",
       "  'tract': 0.007142857142857143,\n",
       "  'bronchiolitis': 0.007142857142857143,\n",
       "  'cough': 0.007142857142857143,\n",
       "  'fever': 0.007142857142857143,\n",
       "  'malaise': 0.007142857142857143,\n",
       "  'symptom': 0.007142857142857143,\n",
       "  'crepitation': 0.014285714285714285,\n",
       "  'wheeze': 0.007142857142857143,\n",
       "  'sign': 0.007142857142857143,\n",
       "  'bronchial': 0.007142857142857143,\n",
       "  'breathing': 0.007142857142857143,\n",
       "  'immunocompromised': 0.007142857142857143,\n",
       "  'likely': 0.007142857142857143,\n",
       "  'nonimmunocompromised': 0.007142857142857143,\n",
       "  'present': 0.007142857142857143,\n",
       "  'versus': 0.007142857142857143,\n",
       "  'p': 0.007142857142857143,\n",
       "  'uneventful': 0.007142857142857143,\n",
       "  'recovery': 0.007142857142857143,\n",
       "  'recovered': 0.007142857142857143,\n",
       "  'following': 0.007142857142857143,\n",
       "  'complication': 0.007142857142857143,\n",
       "  'died': 0.02142857142857143,\n",
       "  'due': 0.007142857142857143,\n",
       "  'underlying': 0.007142857142857143,\n",
       "  'conclusion': 0.007142857142857143,\n",
       "  'similar': 0.007142857142857143,\n",
       "  'published': 0.007142857142857143,\n",
       "  'data': 0.007142857142857143,\n",
       "  'except': 0.007142857142857143,\n",
       "  'finding': 0.007142857142857143,\n",
       "  'mortality': 0.007142857142857143,\n",
       "  'rate': 0.007142857142857143,\n",
       "  'high': 0.007142857142857143},\n",
       " {'nitric': 0.023529411764705882,\n",
       "  'oxide': 0.023529411764705882,\n",
       "  'proinflammatory': 0.011764705882352941,\n",
       "  'mediator': 0.011764705882352941,\n",
       "  'lung': 0.03529411764705882,\n",
       "  'disease': 0.047058823529411764,\n",
       "  'inflammatory': 0.023529411764705882,\n",
       "  'respiratory': 0.023529411764705882,\n",
       "  'tract': 0.023529411764705882,\n",
       "  'commonly': 0.011764705882352941,\n",
       "  'associated': 0.011764705882352941,\n",
       "  'elevated': 0.011764705882352941,\n",
       "  'production': 0.011764705882352941,\n",
       "  'increased': 0.011764705882352941,\n",
       "  'index': 0.011764705882352941,\n",
       "  'dependent': 0.023529411764705882,\n",
       "  'oxidative': 0.011764705882352941,\n",
       "  'stress': 0.011764705882352941,\n",
       "  'although': 0.023529411764705882,\n",
       "  'known': 0.011764705882352941,\n",
       "  'antimicrobial': 0.011764705882352941,\n",
       "  'antiinflammatory': 0.011764705882352941,\n",
       "  'antioxidant': 0.011764705882352941,\n",
       "  'property': 0.011764705882352941,\n",
       "  'various': 0.011764705882352941,\n",
       "  'line': 0.011764705882352941,\n",
       "  'evidence': 0.03529411764705882,\n",
       "  'support': 0.011764705882352941,\n",
       "  'contribution': 0.023529411764705882,\n",
       "  'injury': 0.011764705882352941,\n",
       "  'several': 0.011764705882352941,\n",
       "  'model': 0.011764705882352941,\n",
       "  'basis': 0.011764705882352941,\n",
       "  'biochemical': 0.011764705882352941,\n",
       "  'often': 0.011764705882352941,\n",
       "  'presumed': 0.023529411764705882,\n",
       "  'oxidation': 0.011764705882352941,\n",
       "  'due': 0.011764705882352941,\n",
       "  'formation': 0.011764705882352941,\n",
       "  'oxidant': 0.011764705882352941,\n",
       "  'peroxynitrite': 0.011764705882352941,\n",
       "  'alternative': 0.011764705882352941,\n",
       "  'mechanism': 0.011764705882352941,\n",
       "  'involving': 0.011764705882352941,\n",
       "  'phagocytederived': 0.011764705882352941,\n",
       "  'heme': 0.011764705882352941,\n",
       "  'protein': 0.011764705882352941,\n",
       "  'myeloperoxidase': 0.011764705882352941,\n",
       "  'eosinophil': 0.011764705882352941,\n",
       "  'peroxidase': 0.011764705882352941,\n",
       "  'might': 0.011764705882352941,\n",
       "  'operative': 0.011764705882352941,\n",
       "  'condition': 0.011764705882352941,\n",
       "  'inflammation': 0.011764705882352941,\n",
       "  'overwhelming': 0.011764705882352941,\n",
       "  'literature': 0.011764705882352941,\n",
       "  'generation': 0.011764705882352941,\n",
       "  'activity': 0.011764705882352941,\n",
       "  'would': 0.011764705882352941,\n",
       "  'beyond': 0.011764705882352941,\n",
       "  'scope': 0.011764705882352941,\n",
       "  'commentary': 0.011764705882352941,\n",
       "  'review': 0.011764705882352941,\n",
       "  'area': 0.011764705882352941,\n",
       "  'comprehensively': 0.011764705882352941,\n",
       "  'instead': 0.011764705882352941,\n",
       "  'focus': 0.011764705882352941,\n",
       "  'recent': 0.011764705882352941,\n",
       "  'concept': 0.011764705882352941},\n",
       " {'surfactant': 0.02112676056338028,\n",
       "  'proteind': 0.014084507042253521,\n",
       "  'pulmonary': 0.007042253521126761,\n",
       "  'host': 0.007042253521126761,\n",
       "  'defense': 0.007042253521126761,\n",
       "  'spd': 0.04929577464788732,\n",
       "  'participates': 0.007042253521126761,\n",
       "  'innate': 0.007042253521126761,\n",
       "  'response': 0.014084507042253521,\n",
       "  'inhaled': 0.007042253521126761,\n",
       "  'microorganism': 0.014084507042253521,\n",
       "  'organic': 0.014084507042253521,\n",
       "  'antigen': 0.014084507042253521,\n",
       "  'contributes': 0.014084507042253521,\n",
       "  'immune': 0.007042253521126761,\n",
       "  'inflammatory': 0.007042253521126761,\n",
       "  'regulation': 0.007042253521126761,\n",
       "  'within': 0.007042253521126761,\n",
       "  'lung': 0.028169014084507043,\n",
       "  'synthesized': 0.007042253521126761,\n",
       "  'secreted': 0.007042253521126761,\n",
       "  'alveolar': 0.007042253521126761,\n",
       "  'bronchiolar': 0.007042253521126761,\n",
       "  'epithelial': 0.014084507042253521,\n",
       "  'cell': 0.014084507042253521,\n",
       "  'also': 0.014084507042253521,\n",
       "  'expressed': 0.02112676056338028,\n",
       "  'lining': 0.007042253521126761,\n",
       "  'various': 0.014084507042253521,\n",
       "  'exocrine': 0.007042253521126761,\n",
       "  'duct': 0.007042253521126761,\n",
       "  'mucosa': 0.007042253521126761,\n",
       "  'gastrointestinal': 0.007042253521126761,\n",
       "  'genitourinary': 0.007042253521126761,\n",
       "  'tract': 0.007042253521126761,\n",
       "  'collagenous': 0.007042253521126761,\n",
       "  'calciumdependent': 0.007042253521126761,\n",
       "  'lectin': 0.007042253521126761,\n",
       "  'collectin': 0.007042253521126761,\n",
       "  'bind': 0.014084507042253521,\n",
       "  'surface': 0.02112676056338028,\n",
       "  'glycoconjugates': 0.014084507042253521,\n",
       "  'wide': 0.007042253521126761,\n",
       "  'variety': 0.014084507042253521,\n",
       "  'oligosaccharide': 0.007042253521126761,\n",
       "  'associated': 0.007042253521126761,\n",
       "  'complex': 0.007042253521126761,\n",
       "  'specifically': 0.007042253521126761,\n",
       "  'interacts': 0.007042253521126761,\n",
       "  'molecule': 0.007042253521126761,\n",
       "  'macrophage': 0.014084507042253521,\n",
       "  'neutrophil': 0.007042253521126761,\n",
       "  'lymphocyte': 0.007042253521126761,\n",
       "  'addition': 0.007042253521126761,\n",
       "  'specific': 0.007042253521126761,\n",
       "  'surfactantassociated': 0.007042253521126761,\n",
       "  'lipid': 0.02112676056338028,\n",
       "  'influence': 0.007042253521126761,\n",
       "  'organization': 0.007042253521126761,\n",
       "  'mixture': 0.007042253521126761,\n",
       "  'containing': 0.007042253521126761,\n",
       "  'phosphatidylinositol': 0.007042253521126761,\n",
       "  'vitro': 0.014084507042253521,\n",
       "  'consistent': 0.007042253521126761,\n",
       "  'diverse': 0.007042253521126761,\n",
       "  'activity': 0.007042253521126761,\n",
       "  'observation': 0.007042253521126761,\n",
       "  'spddeficient': 0.014084507042253521,\n",
       "  'transgenic': 0.007042253521126761,\n",
       "  'mouse': 0.014084507042253521,\n",
       "  'show': 0.007042253521126761,\n",
       "  'abnormal': 0.014084507042253521,\n",
       "  'accumulation': 0.014084507042253521,\n",
       "  'respond': 0.007042253521126761,\n",
       "  'abnormally': 0.007042253521126761,\n",
       "  'challenge': 0.007042253521126761,\n",
       "  'respiratory': 0.007042253521126761,\n",
       "  'virus': 0.007042253521126761,\n",
       "  'bacterial': 0.007042253521126761,\n",
       "  'lipopolysaccharides': 0.007042253521126761,\n",
       "  'phenotype': 0.007042253521126761,\n",
       "  'isolated': 0.007042253521126761,\n",
       "  'altered': 0.007042253521126761,\n",
       "  'circumstantial': 0.007042253521126761,\n",
       "  'evidence': 0.007042253521126761,\n",
       "  'oxidant': 0.007042253521126761,\n",
       "  'metabolism': 0.007042253521126761,\n",
       "  'andor': 0.007042253521126761,\n",
       "  'increased': 0.014084507042253521,\n",
       "  'metalloproteinase': 0.007042253521126761,\n",
       "  'expression': 0.014084507042253521,\n",
       "  'development': 0.007042253521126761,\n",
       "  'emphysema': 0.007042253521126761,\n",
       "  'many': 0.007042253521126761,\n",
       "  'form': 0.007042253521126761,\n",
       "  'injury': 0.007042253521126761,\n",
       "  'deficient': 0.007042253521126761,\n",
       "  'appropriately': 0.007042253521126761,\n",
       "  'oligomerized': 0.007042253521126761,\n",
       "  'might': 0.007042253521126761,\n",
       "  'contribute': 0.007042253521126761,\n",
       "  'pathogenesis': 0.007042253521126761,\n",
       "  'human': 0.007042253521126761,\n",
       "  'disease': 0.007042253521126761},\n",
       " {'role': 0.06818181818181818,\n",
       "  'endothelin': 0.045454545454545456,\n",
       "  'lung': 0.06818181818181818,\n",
       "  'disease': 0.11363636363636363,\n",
       "  'et': 0.06818181818181818,\n",
       "  'amino': 0.022727272727272728,\n",
       "  'acid': 0.022727272727272728,\n",
       "  'peptide': 0.022727272727272728,\n",
       "  'diverse': 0.022727272727272728,\n",
       "  'biological': 0.022727272727272728,\n",
       "  'activity': 0.022727272727272728,\n",
       "  'implicated': 0.022727272727272728,\n",
       "  'numerous': 0.022727272727272728,\n",
       "  'potent': 0.022727272727272728,\n",
       "  'mitogen': 0.022727272727272728,\n",
       "  'regulator': 0.022727272727272728,\n",
       "  'smooth': 0.022727272727272728,\n",
       "  'muscle': 0.022727272727272728,\n",
       "  'tone': 0.022727272727272728,\n",
       "  'inflammatory': 0.045454545454545456,\n",
       "  'mediator': 0.022727272727272728,\n",
       "  'may': 0.022727272727272728,\n",
       "  'play': 0.022727272727272728,\n",
       "  'key': 0.022727272727272728,\n",
       "  'airway': 0.022727272727272728,\n",
       "  'pulmonary': 0.022727272727272728,\n",
       "  'circulation': 0.022727272727272728,\n",
       "  'acute': 0.022727272727272728,\n",
       "  'chronic': 0.022727272727272728,\n",
       "  'review': 0.022727272727272728,\n",
       "  'focus': 0.022727272727272728,\n",
       "  'biology': 0.022727272727272728},\n",
       " {'gene': 0.037037037037037035,\n",
       "  'expression': 0.024691358024691357,\n",
       "  'epithelial': 0.024691358024691357,\n",
       "  'cell': 0.024691358024691357,\n",
       "  'response': 0.04938271604938271,\n",
       "  'pneumovirus': 0.024691358024691357,\n",
       "  'infection': 0.037037037037037035,\n",
       "  'respiratory': 0.037037037037037035,\n",
       "  'syncytial': 0.012345679012345678,\n",
       "  'virus': 0.037037037037037035,\n",
       "  'rsv': 0.012345679012345678,\n",
       "  'pneumonia': 0.012345679012345678,\n",
       "  'mouse': 0.012345679012345678,\n",
       "  'pvm': 0.012345679012345678,\n",
       "  'family': 0.012345679012345678,\n",
       "  'paramyxoviridae': 0.012345679012345678,\n",
       "  'subfamily': 0.012345679012345678,\n",
       "  'cause': 0.012345679012345678,\n",
       "  'clinically': 0.012345679012345678,\n",
       "  'important': 0.012345679012345678,\n",
       "  'human': 0.012345679012345678,\n",
       "  'rodent': 0.012345679012345678,\n",
       "  'respectively': 0.012345679012345678,\n",
       "  'target': 0.012345679012345678,\n",
       "  'respond': 0.012345679012345678,\n",
       "  'viral': 0.012345679012345678,\n",
       "  'specific': 0.012345679012345678,\n",
       "  'alteration': 0.012345679012345678,\n",
       "  'including': 0.024691358024691357,\n",
       "  'production': 0.012345679012345678,\n",
       "  'chemoattractant': 0.012345679012345678,\n",
       "  'cytokine': 0.012345679012345678,\n",
       "  'adhesion': 0.012345679012345678,\n",
       "  'molecule': 0.012345679012345678,\n",
       "  'element': 0.012345679012345678,\n",
       "  'related': 0.012345679012345678,\n",
       "  'apoptosis': 0.012345679012345678,\n",
       "  'others': 0.012345679012345678,\n",
       "  'remain': 0.012345679012345678,\n",
       "  'incompletely': 0.012345679012345678,\n",
       "  'understood': 0.012345679012345678,\n",
       "  'review': 0.012345679012345678,\n",
       "  'current': 0.012345679012345678,\n",
       "  'understanding': 0.012345679012345678,\n",
       "  'mucosal': 0.012345679012345678,\n",
       "  'discus': 0.012345679012345678,\n",
       "  'several': 0.012345679012345678,\n",
       "  'genomic': 0.012345679012345678,\n",
       "  'approach': 0.012345679012345678,\n",
       "  'differential': 0.012345679012345678,\n",
       "  'display': 0.012345679012345678,\n",
       "  'reverse': 0.012345679012345678,\n",
       "  'transcriptionpolymerase': 0.012345679012345678,\n",
       "  'chain': 0.012345679012345678,\n",
       "  'reaction': 0.012345679012345678,\n",
       "  'pcr': 0.012345679012345678,\n",
       "  'array': 0.012345679012345678,\n",
       "  'strategy': 0.012345679012345678,\n",
       "  'permit': 0.012345679012345678,\n",
       "  'u': 0.012345679012345678,\n",
       "  'unravel': 0.012345679012345678,\n",
       "  'nature': 0.012345679012345678,\n",
       "  'complete': 0.012345679012345678,\n",
       "  'systematic': 0.012345679012345678,\n",
       "  'manner': 0.012345679012345678}]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_frequency[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28bf4a7d-71b7-407f-a5ed-5a86c13e3244",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "37ef2148-f451-410b-bb79-7ccbddd12d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TF-IDF matrix\n",
    "tfidf_matrix = []\n",
    "for tf in term_frequency:\n",
    "    tfidf_matrix.append({term: tf.get(term, 0) * idf.get(term, 0) for term in unique_terms})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0ebcce00-0e5c-4486-ae3d-33f7a192d743",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert it to DataFrame\n",
    "tfidf_df = pd.DataFrame(tfidf_matrix).fillna(0)\n",
    "tfidf_df.insert(0, 'document_id', list(range(1, len(cleaned_docs) + 1)))\n",
    "\n",
    "# save matrix\n",
    "tfidf_df.to_csv('tfidf_matrix.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "56d98612-f441-497c-8a35-558739730da0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document_id</th>\n",
       "      <th>positive</th>\n",
       "      <th>malaise</th>\n",
       "      <th>method</th>\n",
       "      <th>isolates</th>\n",
       "      <th>hospital</th>\n",
       "      <th>arabia</th>\n",
       "      <th>symptom</th>\n",
       "      <th>due</th>\n",
       "      <th>objective</th>\n",
       "      <th>...</th>\n",
       "      <th>plx</th>\n",
       "      <th>liquid</th>\n",
       "      <th>extension</th>\n",
       "      <th>cystic</th>\n",
       "      <th>describe</th>\n",
       "      <th>sitespecific</th>\n",
       "      <th>handling</th>\n",
       "      <th>alphabet</th>\n",
       "      <th>roomtemperature</th>\n",
       "      <th>manually</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.016447</td>\n",
       "      <td>0.025395</td>\n",
       "      <td>0.007354</td>\n",
       "      <td>0.022499</td>\n",
       "      <td>0.037701</td>\n",
       "      <td>0.050791</td>\n",
       "      <td>0.017548</td>\n",
       "      <td>0.015493</td>\n",
       "      <td>0.022499</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.025518</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2885 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   document_id  positive   malaise    method  isolates  hospital    arabia  \\\n",
       "0            1  0.016447  0.025395  0.007354  0.022499  0.037701  0.050791   \n",
       "1            2  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "2            3  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "3            4  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "4            5  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "\n",
       "    symptom       due  objective  ...  plx  liquid  extension  cystic  \\\n",
       "0  0.017548  0.015493   0.022499  ...  0.0     0.0        0.0     0.0   \n",
       "1  0.000000  0.025518   0.000000  ...  0.0     0.0        0.0     0.0   \n",
       "2  0.000000  0.000000   0.000000  ...  0.0     0.0        0.0     0.0   \n",
       "3  0.000000  0.000000   0.000000  ...  0.0     0.0        0.0     0.0   \n",
       "4  0.000000  0.000000   0.000000  ...  0.0     0.0        0.0     0.0   \n",
       "\n",
       "   describe  sitespecific  handling  alphabet  roomtemperature  manually  \n",
       "0       0.0           0.0       0.0       0.0              0.0       0.0  \n",
       "1       0.0           0.0       0.0       0.0              0.0       0.0  \n",
       "2       0.0           0.0       0.0       0.0              0.0       0.0  \n",
       "3       0.0           0.0       0.0       0.0              0.0       0.0  \n",
       "4       0.0           0.0       0.0       0.0              0.0       0.0  \n",
       "\n",
       "[5 rows x 2885 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d33a20e6-103d-4269-a7ab-fe074d53d0b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_tfidf_file = 'query_tfidf.csv'\n",
    "query_path = 'trec-covid/queries.jsonl'\n",
    "clean_queries='clean_queries.jsonl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a972297b-81f3-4978-aea8-8067425053c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_id': '1', 'text': 'what is the origin of COVID-19', 'metadata': {'query': 'coronavirus origin', 'narrative': \"seeking range of information about the SARS-CoV-2 virus's origin, including its evolution, animal source, and first transmission into humans\"}}\n",
      "{'_id': '2', 'text': 'how does the coronavirus respond to changes in the weather', 'metadata': {'query': 'coronavirus response to weather changes', 'narrative': 'seeking range of information about the SARS-CoV-2 virus viability in different weather/climate conditions as well as information related to transmission of the virus in different climate conditions'}}\n",
      "{'_id': '3', 'text': 'will SARS-CoV2 infected people develop immunity? Is cross protection possible?', 'metadata': {'query': 'coronavirus immunity', 'narrative': 'seeking studies of immunity developed due to infection with SARS-CoV2 or cross protection gained due to infection with other coronavirus types'}}\n",
      "{'_id': '4', 'text': 'what causes death from Covid-19?', 'metadata': {'query': 'how do people die from the coronavirus', 'narrative': 'Studies looking at mechanisms of death from Covid-19.'}}\n",
      "{'_id': '5', 'text': 'what drugs have been active against SARS-CoV or SARS-CoV-2 in animal studies?', 'metadata': {'query': 'animal models of COVID-19', 'narrative': 'Papers that describe the results  of testing drugs that bind to spike proteins of the virus or any other drugs in any animal models. Papers about SARS-CoV-2 infection in cell culture assays are also relevant.'}}\n"
     ]
    }
   ],
   "source": [
    "with open(query_path, 'r', encoding='utf-8') as file:\n",
    "    queries = [json.loads(line) for line in file]\n",
    "\n",
    "\n",
    "for query in queries[:5]:\n",
    "    print(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "348db768-a0ea-4f46-b32d-05800dd6c1f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_queries = []\n",
    "query_dict = {}\n",
    "for query in queries:\n",
    "    query_text = query.get('text', '')\n",
    "    cleaned_tokens = clean_text(query_text)\n",
    "    query_id = query.get('_id', '')\n",
    "    processed_queries.append({'id': query_id, 'text': cleaned_tokens})\n",
    "    query_dict[query_id] = query_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ecc54189-bce7-42af-8eff-44e313fdb0a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(clean_queries, 'w') as file:\n",
    "    for query in clean_queries:\n",
    "        file.write(json.dumps(query) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d014497b-3115-463e-a998-69c5378cced5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': '1', 'text': 'origin covid'}\n",
      "{'id': '2', 'text': 'coronavirus respond change weather'}\n",
      "{'id': '3', 'text': 'sarscov infected people develop immunity cross protection possible'}\n",
      "{'id': '4', 'text': 'cause death covid'}\n",
      "{'id': '5', 'text': 'drug active sarscov sarscov animal study'}\n"
     ]
    }
   ],
   "source": [
    "for query in processed_queries[:5]:\n",
    "    print(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1ab0deea-2912-47d4-83aa-08d96d1d421c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate TF and IDF\n",
    "term_frequency = []\n",
    "query_frequency = Counter()\n",
    "\n",
    "for query in processed_queries:\n",
    "    tokens = query['text'].split()\n",
    "    term_counts = Counter(tokens)\n",
    "    total_terms = len(tokens)\n",
    "    tf = {term: count / total_terms for term, count in term_counts.items()}\n",
    "    term_frequency.append(tf)\n",
    "    query_frequency.update(set(tokens))\n",
    "\n",
    "\n",
    "total_queries = len(processed_queries)\n",
    "idf = {term: math.log(total_queries / (1 + df)) for term, df in query_frequency.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4acfb86d-0e30-49b9-a1ef-4d04c00945d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'origin': 0.5, 'covid': 0.5},\n",
       " {'coronavirus': 0.25, 'respond': 0.25, 'change': 0.25, 'weather': 0.25},\n",
       " {'sarscov': 0.125,\n",
       "  'infected': 0.125,\n",
       "  'people': 0.125,\n",
       "  'develop': 0.125,\n",
       "  'immunity': 0.125,\n",
       "  'cross': 0.125,\n",
       "  'protection': 0.125,\n",
       "  'possible': 0.125},\n",
       " {'cause': 0.3333333333333333,\n",
       "  'death': 0.3333333333333333,\n",
       "  'covid': 0.3333333333333333},\n",
       " {'drug': 0.16666666666666666,\n",
       "  'active': 0.16666666666666666,\n",
       "  'sarscov': 0.3333333333333333,\n",
       "  'animal': 0.16666666666666666,\n",
       "  'study': 0.16666666666666666}]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "term_frequency[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "26afe09d-a515-46cb-aace-6e106db7fce0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create TF-IDF matrix\n",
    "tfidf_queries = []\n",
    "for tf in term_frequency:\n",
    "    tfidf_queries.append({term: tf.get(term, 0) * idf.get(term, 0) for term in unique_terms})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8494e96a-9722-4655-9db3-c9b9138f4967",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to DataFrame\n",
    "tfidf_queries = pd.DataFrame(tfidf_queries).fillna(0)\n",
    "tfidf_queries.insert(0, 'query_id', list(range(1, len(processed_queries)+ 1)))\n",
    "\n",
    "# Save matrix\n",
    "tfidf_queries.to_csv('tfidf_queries.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ad8e56de-c9f9-4135-93e0-319ba607ea7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>positive</th>\n",
       "      <th>malaise</th>\n",
       "      <th>method</th>\n",
       "      <th>isolates</th>\n",
       "      <th>hospital</th>\n",
       "      <th>arabia</th>\n",
       "      <th>symptom</th>\n",
       "      <th>due</th>\n",
       "      <th>objective</th>\n",
       "      <th>...</th>\n",
       "      <th>plx</th>\n",
       "      <th>liquid</th>\n",
       "      <th>extension</th>\n",
       "      <th>cystic</th>\n",
       "      <th>describe</th>\n",
       "      <th>sitespecific</th>\n",
       "      <th>handling</th>\n",
       "      <th>alphabet</th>\n",
       "      <th>roomtemperature</th>\n",
       "      <th>manually</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2885 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id  positive  malaise  method  isolates  hospital  arabia  symptom  \\\n",
       "0         1         0        0       0         0       0.0       0      0.0   \n",
       "1         2         0        0       0         0       0.0       0      0.0   \n",
       "2         3         0        0       0         0       0.0       0      0.0   \n",
       "3         4         0        0       0         0       0.0       0      0.0   \n",
       "4         5         0        0       0         0       0.0       0      0.0   \n",
       "\n",
       "   due  objective  ...  plx  liquid  extension  cystic  describe  \\\n",
       "0    0          0  ...    0       0          0       0         0   \n",
       "1    0          0  ...    0       0          0       0         0   \n",
       "2    0          0  ...    0       0          0       0         0   \n",
       "3    0          0  ...    0       0          0       0         0   \n",
       "4    0          0  ...    0       0          0       0         0   \n",
       "\n",
       "   sitespecific  handling  alphabet  roomtemperature  manually  \n",
       "0             0         0         0                0         0  \n",
       "1             0         0         0                0         0  \n",
       "2             0         0         0                0         0  \n",
       "3             0         0         0                0         0  \n",
       "4             0         0         0                0         0  \n",
       "\n",
       "[5 rows x 2885 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_queries.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51bd9e88-0ac5-4326-8c58-6aefd1b6de26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88eb4d12-4d34-4fde-9c10-de34b4f1446f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f0900085-2281-46f7-b0b3-f6292edd95d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load matrices \n",
    "tfidf_queries = pd.read_csv('tfidf_queries.csv')  \n",
    "tfidf_df = pd.read_csv('tfidf_matrix.csv') \n",
    "# drop the ID columns\n",
    "tfidf_queries_features = tfidf_queries.drop(columns=['query_id'])\n",
    "tfidf_df_features = tfidf_df.drop(columns=['document_id'])\n",
    "# align columns between the two DataFrames\n",
    "all_features = tfidf_queries_features.columns.union(tfidf_df_features.columns)\n",
    "\n",
    "# Reindex both DataFrames to have the same features\n",
    "tfidf_queries_aligned = tfidf_queries_features.reindex(columns=all_features, fill_value=0)\n",
    "tfidf_df_aligned = tfidf_df_features.reindex(columns=all_features, fill_value=0)\n",
    "\n",
    "# compute cosine similarity\n",
    "cosine_sim = cosine_similarity(tfidf_queries_aligned, tfidf_df_aligned)\n",
    "\n",
    "# create a DataFrame\n",
    "cosine_sim_df = pd.DataFrame(cosine_sim, \n",
    "                             index=tfidf_queries['query_id'], \n",
    "                             columns=tfidf_df['document_id'])\n",
    "\n",
    "# save results\n",
    "cosine_sim_df.to_csv('cosine_similarity.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bc92a4f6-6a96-4bf0-8f8d-d57fe7991332",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "document_id   1    2         3    4         5         6         7    8   \\\n",
      "query_id                                                                  \n",
      "1            0.0  0.0  0.000000  0.0  0.000000  0.000000  0.000000  0.0   \n",
      "2            0.0  0.0  0.041253  0.0  0.076283  0.000000  0.000000  0.0   \n",
      "3            0.0  0.0  0.000000  0.0  0.000000  0.000000  0.000000  0.0   \n",
      "4            0.0  0.0  0.000000  0.0  0.057809  0.000000  0.000000  0.0   \n",
      "5            0.0  0.0  0.000000  0.0  0.000000  0.011406  0.019201  0.0   \n",
      "\n",
      "document_id        9    10  ...        61        62   63   64        65  \\\n",
      "query_id                    ...                                           \n",
      "1            0.000000  0.0  ...  0.000000  0.000000  0.0  0.0  0.000000   \n",
      "2            0.000000  0.0  ...  0.000000  0.000000  0.0  0.0  0.090116   \n",
      "3            0.028447  0.0  ...  0.000000  0.000000  0.0  0.0  0.057706   \n",
      "4            0.000000  0.0  ...  0.000000  0.095159  0.0  0.0  0.000000   \n",
      "5            0.000000  0.0  ...  0.018844  0.057754  0.0  0.0  0.000000   \n",
      "\n",
      "document_id       66        67   68        69        70  \n",
      "query_id                                                 \n",
      "1            0.00000  0.000000  0.0  0.000000  0.000000  \n",
      "2            0.00000  0.000000  0.0  0.000000  0.000000  \n",
      "3            0.00000  0.000000  0.0  0.039219  0.000000  \n",
      "4            0.00000  0.000000  0.0  0.000000  0.000000  \n",
      "5            0.01434  0.016356  0.0  0.032006  0.039786  \n",
      "\n",
      "[5 rows x 70 columns]\n"
     ]
    }
   ],
   "source": [
    "print(cosine_sim_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "03a437f9-cd84-4d9d-9a2a-de8aff60bec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# charger les matrices\n",
    "tfidf_queries = pd.read_csv('tfidf_queries.csv')\n",
    "tfidf_df = pd.read_csv('tfidf_matrix.csv')\n",
    "\n",
    "# supprimer les colonnes d'identifiants\n",
    "tfidf_queries_features = tfidf_queries.drop(columns=['query_id'])\n",
    "tfidf_df_features = tfidf_df.drop(columns=['document_id'])\n",
    "\n",
    "# aligner les colonnes entre les deux DataFrames\n",
    "all_features = tfidf_queries_features.columns.union(tfidf_df_features.columns)\n",
    "\n",
    "# réindexer les DataFrames pour avoir les mêmes colonnes/features\n",
    "tfidf_queries_aligned = tfidf_queries_features.reindex(columns=all_features, fill_value=0)\n",
    "tfidf_df_aligned = tfidf_df_features.reindex(columns=all_features, fill_value=0)\n",
    "\n",
    "# Calculer la similarité cosinus\n",
    "cosine_sim = cosine_similarity(tfidf_queries_aligned, tfidf_df_aligned)\n",
    "\n",
    "# Créer un DataFrame pour les similarités\n",
    "cosine_sim_df = pd.DataFrame(cosine_sim, \n",
    "                             index=tfidf_queries['query_id'], \n",
    "                             columns=tfidf_df['document_id'])\n",
    "\n",
    "# Reformater les résultats pour correspondre au format 'test.tsv'\n",
    "output_data = cosine_sim_df.stack().reset_index()\n",
    "output_data.columns = ['query-id', 'corpus-id', 'score']\n",
    "\n",
    "# Enregistrer dans un fichier TSV\n",
    "output_data.to_csv('cosine_similarity_results.tsv', sep='\\t', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d97b86c0-ecb6-4f3e-88dc-c212aa56976e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96283bdd-fcc5-4897-bcb0-dfd22fba42a1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "a1694871-ba9f-4130-8e85-dbc8addf57a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Les résultats ont été sauvegardés dans precision_recall_per_query.csv.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def calculate_precision_recall_per_query(predicted_file, ground_truth_file, output_file):\n",
    "    # charger les résultats prédits\n",
    "    predicted_data = pd.read_csv(predicted_file, sep='\\t')\n",
    "    # charger les résultats de vérité terrain, en s'assurant que le score est numérique\n",
    "    ground_truth_data = pd.read_csv(ground_truth_file, sep='\\t', names=['query-id', 'corpus-id', 'score'])\n",
    "    ground_truth_data['score'] = pd.to_numeric(ground_truth_data['score'], errors='coerce')\n",
    "\n",
    "    # Créer des dictionnaires pour grouper les documents par requête\n",
    "    predicted_dict = predicted_data.groupby('query-id')['corpus-id'].apply(list).to_dict()\n",
    "    ground_truth_dict = ground_truth_data[ground_truth_data['score'] > 0]\\\n",
    "    .groupby('query-id')['corpus-id'].apply(list).to_dict()\n",
    "\n",
    "    results = []\n",
    "\n",
    "    # Parcourir chaque requête dans les résultats de vérité terrain\n",
    "    for query_id in set(ground_truth_dict.keys()).union(predicted_dict.keys()):\n",
    "        relevant_docs = set(ground_truth_dict.get(query_id, []))  # Documents pertinents \n",
    "        retrieved_docs = set(predicted_dict.get(query_id, []))  # Documents prédits\n",
    "\n",
    "        # Calculer précision et rappel\n",
    "        if len(retrieved_docs) > 0:\n",
    "            precision = len(relevant_docs & retrieved_docs) / len(retrieved_docs)\n",
    "        else:\n",
    "            precision = 0\n",
    "\n",
    "        if len(relevant_docs) > 0:\n",
    "            recall = len(relevant_docs & retrieved_docs) / len(relevant_docs)\n",
    "        else:\n",
    "            recall = 0\n",
    "\n",
    "        # Ajouter les résultats pour cette requête\n",
    "        results.append({'query-id': query_id, 'precision': precision, 'recall': recall})\n",
    "\n",
    "    # Convertir les résultats en DataFrame\n",
    "    results_df = pd.DataFrame(results)\n",
    "\n",
    "    # Sauvegarder les résultats dans un fichier CSV\n",
    "    results_df.to_csv(output_file, index=False)\n",
    "\n",
    "\n",
    "\n",
    "# Fichiers d'entrée\n",
    "predicted_file = 'cosine_similarity_results.tsv'  \n",
    "ground_truth_file = 'trec-covid/test.tsv'  \n",
    "output_file = 'precision_recall_per_query.csv'  \n",
    "\n",
    "# calcul des métriques par requête\n",
    "calculate_precision_recall_per_query(predicted_file, ground_truth_file, output_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37469ff-411d-4e4a-b1de-ba9f88ce782f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3176f92b-f240-4214-bd8b-52e2fc70992b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# expand query using synonyms\n",
    "\n",
    "def expand_query(query):\n",
    "    synonyms = set()\n",
    "    for word in word_tokenize(query):\n",
    "        for syn in wordnet.synsets(word):\n",
    "            for lemma in syn.lemmas():\n",
    "                synonyms.add(lemma.name().replace('_', ' '))\n",
    "                if len(synonyms) >= 5: \n",
    "                    break\n",
    "    return ' '.join(synonyms.union(query.split()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e0b05c2a-72af-4ed4-9c34-fe62b8860099",
   "metadata": {},
   "outputs": [],
   "source": [
    "# search and rank results >> top 10\n",
    "def show_results(query):\n",
    "    expanded_query = expand_query(query)  # expand the query\n",
    "    expanded_tokens = clean_text(expanded_query).split()  \n",
    "    query_vec = pd.DataFrame([{term: expanded_tokens.count(term) * idf.get(term, 0) for term in unique_terms}])\n",
    "\n",
    "    # ensure alignment with TF-IDF matrix\n",
    "    query_vec = query_vec.reindex(columns=tfidf_df.columns[1:], fill_value=0)  \n",
    "    similarities = cosine_similarity(query_vec.values, tfidf_df.drop(['document_id'], axis=1).values)\n",
    "\n",
    "    # print top 10 relevence docs\n",
    "    sorted_results = pd.Series(similarities[0], index=tfidf_df['document_id']).sort_values(ascending=False).head(10)\n",
    "    results_text.delete('1.0', tk.END)\n",
    "\n",
    "    # result dictionary\n",
    "    results = {}\n",
    "    for doc_id in sorted_results.index:\n",
    "        doc_title = cleaned_docs[doc_id - 1]['title']  # docu title\n",
    "        doc_real_id = cleaned_docs[doc_id - 1]['id']  # document id\n",
    "        query_id = [key for key, value in query_dict.items() if value == query]  # find the query id\n",
    "        if query_id:\n",
    "            query_id = query_id[0]\n",
    "            results.setdefault(query_id, []).append(doc_real_id)\n",
    "        results_text.insert(tk.END, f\"{doc_real_id} - {doc_title}\\n\")  # print title and id\n",
    "\n",
    "    \n",
    "    print(f\"Results for Query '{query}': {results}\")\n",
    "\n",
    "    # Calculate and display precision/recall\n",
    "    try:\n",
    "        precision, recall = calculate_precision_recall('trec-covid/test.tsv', results)  # Use ground truth file for evaluation\n",
    "        results_text.insert(tk.END, f\"\\nPrecision: {precision:.2f}\\nRecall: {recall:.2f}\\n\")\n",
    "    except Exception as e:\n",
    "        results_text.insert(tk.END, f\"\\nError calculating precision/recall: {e}\\n\")\n",
    "        print(f\"Error in precision/recall calculation: {e}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a33737-28dc-4091-89ab-76fc15a09281",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for Query 'what is the origin of COVID-19': {'1': ['6iu1dtyl', 'eiqypt0m', 'yba7mdtb', 'ejv2xln0', 'ug7v899j', '02tnwd4m', 'zjufx4fo', '9785vg6d', '2b73a28n', '5yhe786e']}\n",
      "Error in precision/recall calculation: Invalid file path or buffer object type: <class 'dict'>\n",
      "Results for Query 'will SARS-CoV2 infected people develop immunity? Is cross protection possible?': {'3': ['j3p1u80n', 'bbvxu8op', 'pk7pnmlo', 'utglk4af', 'di0fcy0j', 'e62cfqt7', 'qva0jt86', '1r65yam5', '4k8f7ou1', 't40ybhgb']}\n",
      "Error in precision/recall calculation: Invalid file path or buffer object type: <class 'dict'>\n",
      "Results for Query 'what is the origin of COVID-19': {'1': ['6iu1dtyl', 'eiqypt0m', 'yba7mdtb', 'ejv2xln0', 'ug7v899j', '02tnwd4m', 'zjufx4fo', '9785vg6d', '2b73a28n', '5yhe786e']}\n",
      "Error in precision/recall calculation: Invalid file path or buffer object type: <class 'dict'>\n",
      "Results for Query 'corona virus': {}\n",
      "Error in precision/recall calculation: Invalid file path or buffer object type: <class 'dict'>\n"
     ]
    }
   ],
   "source": [
    "import tkinter as tk\n",
    "from tkinter import ttk\n",
    "\n",
    "# Main interface setup\n",
    "root = tk.Tk()\n",
    "root.title(\"IR System\")\n",
    "root.geometry(\"800x500\")\n",
    "\n",
    "# Basic styling\n",
    "root.configure(bg=\"#F0F0F0\")\n",
    "\n",
    "query_var = tk.StringVar()\n",
    "\n",
    "# Label and Combobox for predefined queries\n",
    "combo_label = tk.Label(root, text=\"Select Query:\", font=(\"Arial\", 10), bg=\"#F0F0F0\")\n",
    "combo_label.grid(row=0, column=0, padx=10, pady=5, sticky=\"w\")\n",
    "combo = ttk.Combobox(root, textvariable=query_var, values=list(query_dict.values()), width=50)\n",
    "combo.grid(row=0, column=1, padx=10, pady=5)\n",
    "combo.bind(\"<KeyRelease>\", lambda event: combo.event_generate('<<ComboboxSelected>>'))\n",
    "\n",
    "# Label and Entry for manual query input\n",
    "entry_label = tk.Label(root, text=\"Manual Query:\", font=(\"Arial\", 10), bg=\"#F0F0F0\")\n",
    "entry_label.grid(row=1, column=0, padx=10, pady=5, sticky=\"w\")\n",
    "entry = tk.Entry(root, textvariable=query_var, width=50)\n",
    "entry.grid(row=1, column=1, padx=10, pady=5)\n",
    "\n",
    "# Search button\n",
    "search_button = tk.Button(root, text=\"Search\", command=lambda: show_results(query_var.get()), bg=\"#0078D7\", fg=\"white\", font=(\"Arial\", 10))\n",
    "search_button.grid(row=0, column=2, rowspan=2, padx=10, pady=5, sticky=\"ns\")\n",
    "\n",
    "# Results text box\n",
    "results_label = tk.Label(root, text=\"Results:\", font=(\"Arial\", 10), bg=\"#F0F0F0\")\n",
    "results_label.grid(row=2, column=0, padx=10, pady=5, sticky=\"nw\")\n",
    "results_text = tk.Text(root, height=100, width=100, bg=\"#FFFFFF\", fg=\"black\", font=(\"Arial\", 10))\n",
    "results_text.grid(row=3, column=0, columnspan=3, padx=10, pady=5)\n",
    "\n",
    "root.mainloop()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59351eb2-68f2-4b69-a1e9-3a22d8af607c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71caf60f-301d-4859-8c20-91a285ff62a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}