SALAMI_CHECKER.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Team              : CTRL C CTRL V\n",
        "# Track Company     : Track-1 Cactus Communications\n",
        "# Track Name        : Paperpal - Future of Academic Writing\n",
        "# Chec Type         : Content Centric Checks\n",
        "# Check Implemented : Salami Publishing Check"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Salient Features : \n",
        "\n",
        "1. The author name is searched on the google scholar and every research paper, available publicly is downloaded.\n",
        "2. By far, the most complicated and useful check implemented using best preprocessing techniques, which skims the abstract, introduction and conclusion of every research paper of an author. \n",
        "3. This then goes through series of checks, which summarizes the text and uses Doc2Vec model for further processing.\n",
        "4. The model then gives output as similarity index with the mentioned pdf, which gives us an idea of author's work and how they are similar to each other."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bs-t_F5BS-VR",
        "outputId": "c1592218-e42d-45c6-9c64-a5adb3647e6c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting PYPDF2\n",
            "  Downloading pypdf2-3.0.1-py3-none-any.whl (232 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m232.6/232.6 KB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing_extensions>=3.10.0.0 in /usr/local/lib/python3.8/dist-packages (from PYPDF2) (4.5.0)\n",
            "Installing collected packages: PYPDF2\n",
            "Successfully installed PYPDF2-3.0.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Spacy in /usr/local/lib/python3.8/dist-packages (3.4.4)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.8/dist-packages (from Spacy) (3.0.12)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (1.22.4)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from Spacy) (0.10.1)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from Spacy) (2.0.7)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (0.7.0)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (1.0.4)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from Spacy) (1.10.5)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from Spacy) (3.0.8)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from Spacy) (6.3.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (1.0.9)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from Spacy) (57.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from Spacy) (3.1.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (3.3.0)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (8.1.7)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from Spacy) (2.0.8)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/dist-packages (from Spacy) (0.10.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (2.25.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from Spacy) (23.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from Spacy) (2.4.6)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->Spacy) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->Spacy) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->Spacy) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->Spacy) (1.26.14)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->Spacy) (4.0.0)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->Spacy) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->Spacy) (0.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->Spacy) (8.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/dist-packages (from jinja2->Spacy) (2.1.2)\n"
          ]
        }
      ],
      "source": [
        "!pip install PYPDF2\n",
        "!pip install Spacy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P_4KoHG0ZF0_"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "file_size = os.path.getsize('/content/CoCoLe_2023_paper_23.pdf')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ML1pMyUtTQoI",
        "outputId": "3abf693e-500b-470b-9776-aa244e249227"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[' . The accessibility internet Geographic Informa- tion Systems ( GIS ) significantly affected people lived daily lives . These systems using various GIS data . There two types GIS data : raster data vector data . When working contin- uous data types elevation satellite imagery , raster data useful . The existing methods convert raster data vector data processing , results loss details , loss accuracy , changes original data . This research focuses finding ways improve processing analysis large raster datasets without compromising precision causing significant loss . In solution pro- posed , capabilities GeoTrellis Docker utilized . GeoTrellis , support various raster data storage formats ability process data without conversion , well-suited handle large-scale data processing . By using Docker distribute computation across cluster , solution able efficiently process complex tasks reasonable amount time . A collection algorithms handle data hasten output ad-hoc queries also offered proposed system architecture . ', ' Future Work The paper introduces Geotrellis framework analyzing large-scale geospa- tial raster data performed Docker container achieve system independence . Our system tested using two nodes Hadoop Spark clusters , andScalable Analytics Raster Data Processing ... 13 determined operates effectively within frameworks . Tile- based processing carried lessen computer cluster load instead analyzing entire dataset . The default block size Hadoop 128 MB , observed affect masking time processing files different sizes , highlighted Table 1 . For example , consider cases 9 MB 30 MB , clearly see total time 9 MB higher compared 30 MB . So , address , adjust default size Hadoop . The time increases dramatically 610 MB well . To address issue , increase number nodes . In order determine efficiency strategy , intend expand number nodes compare masking times vari- ous file sizes future work . Additionally , intend upgrade cluster commodity hardware run system diverse data set . Acknowledgement This work supported Indian Space Research Organisation ( ISRO ) , Government India , RESPOND program . Also thankful ISRO focal person Mr. Ghansham Sangar Mr. Pankaj Bodani constant support project development . Declaration –Conflicts Interests : The authors declare conflicts sort associated paper . –Data Availability Statement : Data sharing applicable article datasets generated analyzed current study . ', ['Spark GeoTrellis', 'Smiti Kothari', 'Jayneel Shah', 'JaiPrakash Verma', 'Sapan H Mankad']]\n"
          ]
        }
      ],
      "source": [
        "import PyPDF2\n",
        "import spacy\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize \n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "# --- main ---\n",
        "def getACA(file):\n",
        "  pdfFileObj = open(file, 'rb')\n",
        "\n",
        "  pdfReader = PyPDF2.PdfReader(pdfFileObj)\n",
        "\n",
        "  text = ''\n",
        "\n",
        "  num_pages = len(pdfReader.pages)\n",
        "      \n",
        "  for page_num in range(num_pages):\n",
        "    page = pdfReader.pages[page_num]\n",
        "    page_text = page.extract_text()\n",
        "    text += page_text\n",
        "\n",
        "  stop_words = set(stopwords.words('english')) \n",
        "\n",
        "# tokens of words  \n",
        "  word_tokens = word_tokenize(text) \n",
        "    \n",
        "  filtered_sentence = [] \n",
        "  \n",
        "  for w in word_tokens: \n",
        "    if w not in stop_words: \n",
        "        filtered_sentence.append(w) \n",
        "#print(\"\\n\\nOriginal Sentence \\n\\n\")\n",
        "#print(\" \".join(word_tokens)) \n",
        "  text=\" \".join(filtered_sentence)\n",
        "#print(\"\\n\\nFiltered Sentence \\n\\n\")\n",
        "#print(\" \".join(filtered_sentence))\n",
        "\n",
        "  pos = text.lower().find('abstract')\n",
        " \n",
        "  \n",
        "\n",
        "  pos1 = text.lower().find('keywords')\n",
        "  text1 = text[pos+len('abstract'):pos1]\n",
        "  \n",
        "  #print(text1)\n",
        "  # For Conclusion\n",
        "\n",
        "  pos2 = text.lower().find('conclusion')\n",
        "  pos3 = text.lower().find('references')\n",
        "  text2 = text[pos2+len('conclusion'):pos3]\n",
        "  #print(text2)\n",
        "\n",
        "  # For Author\n",
        "  text3=text[:pos]\n",
        "  #print(text3)\n",
        "  char_str = '' .join((z for z in text3 if not z.isdigit()))\n",
        "  #print(char_str) \n",
        "  text5 = []\n",
        "  doc=nlp(char_str)\n",
        "  \n",
        "\n",
        "  list1=[]\n",
        "  for ent in doc.ents:\n",
        "    if ent.label_ == 'PERSON':\n",
        "      text4=ent.text\n",
        "      text5.append(text4)\n",
        "      # print(text5)\n",
        "  if(pos == file_size):\n",
        "    text1=[]\n",
        "    text2=[]\n",
        "    text5=[]\n",
        "  list1.append(text1)\n",
        "  \n",
        "  list1.append(text2)\n",
        "  list1.append(text5)\n",
        "  return list1\n",
        "\n",
        "vals = getACA(\"/content/CoCoLe_2023_paper_23.pdf\")\n",
        "main_abs = vals[0]\n",
        "main_concl = vals[1]\n",
        "main_string = main_abs + ' ' + main_concl\n",
        "\n",
        "print(vals)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HgZAgwjSUHQH",
        "outputId": "eb42b855-3b56-471f-81ff-c56e97f2c9be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['https://docs.huihoo.com/apache/spark/summit/2014/Geotrellis-Adding-Geospatial-Capabilities-to-Spark-Ameet-Kini-Rob-Emanuele.pdf', 'https://jiayuasu.github.io/files/paper/geospatial-icde-2019.pdf', 'https://agile-giss.copernicus.org/articles/1/10/2020/agile-giss-1-10-2020.pdf', 'https://vikalpsangam.org/wp-content/uploads/migrate/Resources/niyamgiricasestudyjan2018.pdf', 'https://vikalpsangam.org/wp-content/uploads/migrate/Resources/niyamgiricasestudyjan2018.pdf', 'https://ashishkothari.in/wp-content/uploads/2022/10/Revisiting-the-legend-of-Niyamgiri-The-Hindu.pdf', 'https://ashishkothari.in/wp-content/uploads/2022/10/Revisiting-the-legend-of-Niyamgiri-The-Hindu.pdf', 'https://link.springer.com/content/pdf/10.1007/978-981-13-1747-7.pdf', 'http://pages.cs.wisc.edu/~jayneel/papers/ieeemicro11_fabscalar.pdf', 'https://www.academia.edu/download/38018754/anubhuti_paper.pdf', 'https://www.researchgate.net/profile/Jaiprakash-Verma/publication/282686173_Big_Data_Analysis_Recommendation_System_with_Hadoop_Framework/links/57f4afb708ae280dd0b77681/Big-Data-Analysis-Recommendation-System-with-Hadoop-Framework.pdf', 'https://www.academia.edu/download/37430843/Meena_et_al.__2015_EE_KSR.pdf', 'https://www.academia.edu/download/68592157/j.ecoleng.2012.12.02220210804-6629-9c9zpo.pdf', 'https://www.researchgate.net/profile/Vijay-Meena-2/publication/318291832_Plant_beneficial_rhizospheric_microorganism_PBRM_strategies_to_improve_nutrients_use_efficiency_A_review/links/5a0422c70f7e9beb1774ed50/Plant-beneficial-rhizospheric-microorganism-PBRM-strategies-to-improve-nutrients-use-efficiency-A-review.pdf']\n"
          ]
        }
      ],
      "source": [
        "import requests\n",
        "import nltk \n",
        "from bs4 import BeautifulSoup\n",
        "from urllib.request import urlopen\n",
        "\n",
        "def download_pdfs(links):\n",
        "    for i, link in enumerate(links):\n",
        "        response = requests.get(link)\n",
        "        new_name = f\"{i+1}.pdf\"\n",
        "        # filename = link.split('/')[-1]\n",
        "        with open(new_name, 'wb') as f:\n",
        "            f.write(response.content)\n",
        "\n",
        "def crawl_google_scholar(author_name):\n",
        "    url = f'https://scholar.google.com/scholar?q={author_name}'\n",
        "    response = requests.get(url)\n",
        "    soup = BeautifulSoup(response.text, 'html.parser')\n",
        "    \n",
        " \n",
        "    #print(text)\n",
        "    links = soup.find_all('a', href=True)\n",
        "    pdf_links = [link['href'] for link in links if link['href'].endswith('.pdf')]\n",
        "    # print(pdf_links)\n",
        "    return pdf_links\n",
        "pdf_links=[]\n",
        "for name in vals[2]:\n",
        "  pdf_links += crawl_google_scholar(name)\n",
        "print(pdf_links)\n",
        "download_pdfs(pdf_links)\n",
        "# for link in pdf_links:\n",
        "#     print(link)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5bsm8dOnVgtb",
        "outputId": "e4edb6d8-a57e-4784-fc84-7fe36d4ad4aa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " . We present cloud-based approach transform arbitrar- ily large terrain data hierarchical level-of-detail structure optimized web visualization . Our approach based divide-and-conquer strategy . The input data split tiles distributed toindividual workers cloud . These workers apply Delaunay trian-gulation maximum number points maximum geometricerror . They merge results triangulate generateless detailed tiles . The process repeats hierarchical tree diﬀer-ent levels detail created . This tree used streamthe data web browser . We implemented approach theframeworks Apache Spark GeoTrellis . Our paper includes evalu-ation approach implementation . We focus scalabilityand runtime also investigate bottlenecks , possible reasons , well options mitigation . The results evaluation show thatour approach implementation scalable able toprocess massive terrain data . \n",
            "s directions futurework ( Section 6 ) . 2 Related work Processing geospatial data cloud become importantto research community last decade . This section divided threesubsectionsthatpresentapproacheswithregardtogeneralcloudprocessing ( Sec-tion 2.1 ) , distributed geo processing algorithms cloud ( Section 2.2 ) , andattempts parallelize Delaunay triangulation ( Section 2.3 ) . 2.1 General cloud processing approaches General purpose architectures developed able deploy dif- ferent processing algorithms . Two major approaches batch stream pro-cessing . Batch processing works well existing datasets acquired certain time later transformed . Scientiﬁc workﬂow managementsystems Pegasus [ 10 ] Kepler [ 25 ] support style processing.The applies architecture presented earlier work [ 18,17 ] theMapReduce programming paradigm [ 9 ] . For constant stream incoming data , stream processing developed . In stream processing , implemented Apache Spark Streaming [ 3 ] Storm [ 1 ] , data processed immediately acquired . The result dataset isupdatedincrementally.Asageneraldownside , stream processing introducessome overhead . To mitigate , novel concepts micro-batching , h eLambda architecture [ 26 ] , Kappa architecture [ 19 ] emerged . Isenburg et al . shown high potential streaming triangula- tion spatially large areas respect memory time consumption [ 16 ] .In contrast approach , method lacks ability introduce error 2 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.bound metric triangulation ability parallelize triangulation processes . Our approach typical batch process expecting data present beginning . 2.2 Geo-speciﬁc cloud approaches In addition general-purpose architectures mentioned Section 2.1 , areapproachesspecializedfortheprocessingofgeospatialdata.Theyarestronglycoupled underlying cloud infrastructure . For example , Qazi et al . describea software architecture model domestic wastewater treatment solutions Ire-land [ 31 ] . Their solution depends Amazon Web Services installthe commercial tool ArcGIS Server via special Amazon Machine Images ( AMIs ) provided Esri . Warren et al . process petabyte data acquired theUS Landsat MODIS programs past 40 years [ 37 ] . Their processingpipeline connects 10 steps including uncompressing raw image data , classiﬁca-tion points , cutting tiles , performing coordinate transformations , storingthe results Google Cloud Storage . Their static process highly optimizedfor Google platform . Li et al . use Microsoft Azure infrastructure pro-cess high-volume datasets satellite imagery [ 22 ] . They leverage cluster of150 virtual machine instances making process 90 times faster comparisonto conventional application high-end desktop machine . In contrast works , approach depend speciﬁc cloud infrastructure . We require GeoTrellis , installed arbitrary ( virtual ) machines , even cluster grid . We think GeoTrellis ( andthe underlying Spark framework ) good choice processing largegeospatial data . This supported work Liu et al . present anapproach detect changes large LiDAR point clouds [ 24 ] . They summarizethat Spark suitable process data exceeds capacities typicalGIS workstations , matches evaluation results regard datascalability . 2.3 Parallelized Delaunay triangulation Other earlier works focus parallelizing Delaunay triangulation . Spielman et al . presented modiﬁcation delaunay triangulation enables par-allel mesh updates [ 34 ] . In iteration , choose npoints insert mesh parallelized manner . Hu et al . go even map thetriangulation GPU-based implementation [ 15 ] . For , load modelin video memory compute updates vertex stream geometryshaders . With approach , able generate real-time view-dependentmeshes , long models small enough . Concerning scalable creationof TINs , Goodrich presents method create convex hull Delaunaytriangulation performed bulk-synchronous parallel computers [ 13 ] . A re-cent approach Nath et al . guarantees O ( nl gn ) runtime create TIN DEMs 3 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.with modiﬁed Delaunay algorithm tailored massive parallel communi- cation model [ 27 ] . In contrast , approach distributes terrain tiles multiple virtual machine instancesinthecloudinordertodividethetime-consumingtriangulationprocessto available compute resources . On instance , triangulation anon-parallel iterative process . 3 Approach Our approach based strong mapping target data structureand processing distributed cloud . We look geospatialdataisusuallyorganizedforweb-basedvisualizationandoptimizeourprocessingand distribution strategy accordingly . 3.1 Hierarchical level-of-detail structure terrain Ulrich shown hierarchical Level Detail ( LoD ) structures allow ar- bitrarily sized data visualized web [ 36 ] . Formats I3S [ 12 ] 3D Tiles [ 6 ] follow approach optimized 3D objects notterrain . As mentioned , use Cesium ( Version 1.64 ) web visual-ization . This framework supports Quantized Mesh ( QM ) format [ 5 ] , whichis similar data format optimized terrain . In QM , globe divided intoquadratic subsections ( tiles ) diﬀerent granularity according given zoomlevel . These tiles organized hierarchical quadtree speciﬁed TileMap Service ( TMS ) using global-geodetic proﬁle [ 30 ] . This tiling scheme similar one used Web Map Tiling Service ( WMTS ) [ 28 ] . Fig . 1.Layout scheme zoom levels 0 1 . With increasing levels , resolution increases . Image earth [ 35 ] . 4 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 2.Depending camera position , Cesium displays diﬀerent levels detail . In left picture , lower resolution loaded right picture . QM initially divides earth two tiles zoom level 0 . For next level , eachtilefromthepreviouslevelisdividedintofoursubtiles.Inthisprocess , number points stored mesh increases . In level 0 , error ( i.e . thediﬀerence ) provided mesh real earth surface high , whilelevel17providesenoughdetailtomodel objectswithasizeofafewmeters.The dividing process tiles repeated subsequent zoom levels , resultingin hierarchical subdivision surface earth ( see Figure 1 ) . The QMterrain format additionally speciﬁes mesh data individual tilesshould saved . When globe ’ model displayed , Cesium determines tiles loaded based current view . This restricted single zoom level.Instead , Cesium mixes tiles diﬀerent levels depicted Figure 2 . 3.2 Divide-and-conquer strategy scalable processing Let us assume want generate terrain meshes raster dataset Don level within range bottom zoom level b∈Nto top zoom level t∈Nwith b > . In order achieve scalability , apply divide-and-conquer strategy split input data tiles processed individually separate nodes cloud . The tile layout determined target format , , described , hierarchical quadtree . The process illustrated Figure 3 . First , need ﬁnd pixels fromDcorrespond tiles output data structure . For , split resample Dinto raster tiles according bottom level bof layout scheme . This includes loading repartitioning source data ( seeSection 4.1 ) well applying layout scheme ( Step 1 ) . The repartitioned tiles extent terrain mesh tiles expected Cesium forthe bottom level band contain height data resampled pixel values . In next step 2 , convert 2.5D raster tiles set 3D points triangulate described Section 4.3 . This results inmesh tiles containing height information corresponding extent 5 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Raster Tiles Mesh TilesAggregated Points TileMesh Tile aggregationtriangulation triangulationiterationSource GeoTi \u0001 FilesGeoTrellis Triangulationloading , repartition applying layout schema level bD1 2 3 45 Fig . 3.Basic concept conversion process raster input ﬁles meshes required levels detail . form geometry . The following steps conversion process get height information mesh instead original raster data ( bottom row inFigure 3 ) . When mesh tiles zoom level bare generated , aggregate mesh data according layout zoom level b−1 3 . In case , tile level b−1 four subtiles level bto merged together described Section 4.4 . To achieve , ﬁrst extract point data meshes . Afterwards , start new triangulation process points higher maximumerror 4 . This results new mesh lower resolution tile zoom level b−1 . This procedure repeated every zoom level minimum level tis converted completely 5 . 4 Implementation Our main idea map hierarchical conversion problem data structurescompatible Apache Spark [ 2 ] ( Version 2.20 ) , provides abilitytotriangulatetheheightdatainadistributedwayandutilizetheresourcesofthecloud infrastructure . For triangulation , explain handle rasterdata GeoTrellis [ 4 ] ( Version 1.2.1 ) merging less detailedlevels done . Apache Spark framework distributed computing . It consists multiple components diﬀerent responsibilities : – Master Manages available workers executors distributes request driver . – Worker A server instance consisting multiple executors . 6 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.– Executor A working unit using multiple cores . It perform necessary calculations task . –D r v e r Theprogramwherethetasksarecreated.Thedriverasksthemaster executors sendstasks tothem . This allowsmultiple driversto request calculation capabilities master . Apache Spark enables us build scalable platform distributed calcu- lations . It hides network activities executor allocation , focuson program logic . GeoTrellis framework processing georeferenced raster data . Forthis , itprovidesdatatypes , I/Ofunctionality , andrastermappingoperations . We use GeoTrellis read information multiple input ﬁles , manage underlying coordinate reference system , produce normalized tiles.A tile rectangle customizable dimensions covering speciﬁc area inthe source data . For example , possible split input data ntiles , consisting width×heightpixels . Based layout scheme Figure 1 , use width=height= 256 . This means single tile mesh generated based raster 256 ×256 pixels . GeoTrellis map , ﬁlter , manipulate tiles . GeoTrellis integrates Apache Spark . This allows us combine ben- eﬁts managed distributed computation support geospatial data . 4.1 Loading repartitioning source raster data We use GeoTrellis split input terrain Dinto separate tiles ( Figure 3 , Step 1 ) . GeoTrellis creates so-called Resilient Distributed Dataset ( RDD ) contains individual tiles directly used Spark . To guarantee RDD tile size 256 ×256 pixels make use resampling methods oﬀered GeoTrellis—i.e . Nearest Neighbor ( NN ) andBilinear sampling ( see Section 5.2 ) . The whole mapping process executed Spark cluster . The terrain meshes generated individual Spark instances saved hard drive . 4.2 Calculating required bottom zoom level Algorithm 1 determines appropriate bottom zoom level bfor given input tile resolution.ThealgorithmcalculatesthewidthandheightofonesourceGeoTIFF ﬁle delta top-left bottom-right WGS84 coordinates assumingthat GeoTIFF ﬁles resolution . It divides values theﬁle size pixels multiplies grid size , results areathat covered tile ﬁnal output . The algorithm increases zoomlevel incrementally checks whether ﬁner required granularity.It returns ﬁrst level suﬃcient cover data included sourceraster . 7 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Algorithm 1 Calculate required zoom level 1 : Input : One GeoTIFF tfrom dataset , 2 : The Gridsize sof one output cell pixels 3 : procedure requiredZoom ( GeoTIFF , Gridsize ) 4 : ⊿One ﬁnal cell least cover area 5 : requiredWidth =t.extent.width/t.width ∗s 6 : requiredHeight =t.extent.height/t.height ∗s 7 : requiredSize = ( requiredWidth , requiredHeight ) 8 : 9 : center=t.middle 10 : zoom=0 11:12 : whilecellAt ( zoom , center ) .extent > requiredSize 13 : zoom=zoom+1 14 : end 15:16 : returnzoom ⊿ First level data loss 17 : end procedure 4.3 Raster triangulation In Steps 2a n d4 conversion process ( Figure 3 ) , apply Delaunay triangulation sets points extracted input raster data generated mesh tiles . Our implementation triangulation algorithm followsthe iterative approach presented de Berg et al . [ 8 ] uses diﬀerent strategyto select points added mesh ( see Algorithm 2 ) . First , specify corner points C , four points four corners tile . They deﬁnitely included resulting meshin order make sure covers full size tile holes theﬁnal rendering avoided . The algorithm starts mesh consists oftwo triangles based four points C. In process , points iteratively integrated mesh vertices ( line 13 ) . For , pointwith maximum distance mesh extracted line 12 . This approach isbased idea adding point maximum distance results ahigh increase quality resulting mesh ( see also [ 21 ] ) . For point , height compared heightAt ( , p ) Line 12 . This function calculates height mesh position Point p. The point maximum distance ( argmax ) mesh added . 3 This done one given termination conditions met . The param- eter maxPoints deﬁnes maximum number points use resulting mesh . As soon number reached , triangles added tothe mesh . The second termination condition given maxError . The algo- rithm terminate maximum distances original points 3Notethataddingapointtothemeshmayrequireedgeﬂippingtoensurethetriangles still meet Delaunay condition . Details beyond scope paper.We refer original algorithm de Berg et al . [ 8 ] . 8 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Algorithm 2 Triangulation one tile 1 : Input : Set Points Pwhich triangulated , 2 : Corner Points Cwhich included deﬁnitely , 3 : maxError allows quality based termination , 4 : maxPoints limit resulting mesh size 5 : Output : Meshmapproximating given points 6 : procedure triangulate ( P , C , maxError , maxPoints ) 7 : ⊿The initial Mesh covers corner points 8 : Mesh m=newMesh ( C ) 9 : error=calculateError ( , P ) 10 : 11 : whileerror > maxError andm.points.length < maxPoints 12 : point=argmaxPo n p ∈P|p.height −heightAt ( , p ) | 13 : m.addPoint ( point ) 14 : error=calculateError ( , P ) 15 : end 16:17 : returnm 18 : end procedure 19 : ⊿Calculate max distance Mesh point P 20 : function calculateError ( Mesh , P n sP ) 21 : returnmax Po n p ∈P|p.height −heightAt ( , p ) | 22 : end function mesh ( calculated function calculateError ( , P ) ) less maxError . In case , adding new point would exceed desired quality maxError . An error 100 meters mesh real height suﬃcient whole globe displayed , zoom , lowerthe value . We calculate maxError dynamically based following empirical formula : maxError =150000 2zoomLevel On zoom level 0 , maxError 150 kilometers . This value halved every increase zoom level . 4.4 Merging mesh tiles In aggregation step ( Figure 3 , Step 3 ) , merge four tiles level bto generate input data triangulating next , less detailed level b−1i ns u c ha way necessary resample whole dataset Dagain . Based layout scheme , tile size 256 ×256 pixels . Merging four tiles results new tile size [ 0 , 511 ] 2 . We resample tile [ 0 ,255 ] 2 achieve uniform tile format . After merging step , perform triangulation described Algo- rithm 2 get new mesh ( Step 4 ) . The algorithm requires four corner 9 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 4.A screenshot triangulated test dataset : terrain model textures front wireframe back . pointsto start ( see Section 4.3 ) . They extracted corner points level . This process repeats levels generated ( Step 5 ) . Figure 4 shows screenshot ﬁnal result : triangulated test dataset visualized web browser Cesium . 5 Evaluation In section , present results evaluating approach im-plementation based test dataset containing 973 terrain tiles ( stored theGeoTIFF ﬁle format ) covering whole German Federal State Hesse.Each tiles resolution 5000 ×5000 pixels one pixel per square meter , results total area 24325 km 2 . The total data size 84 GB . ThecopyrightofthedatasetisheldbytheHessianStateOﬃceforLandManage-ment Geo-Information ( HVBG ) . Publishing data allowed , itcan acquired online portal organization [ 14 ] . As mentionedin Section 4 , use grid size 256 pixels layout scheme . According toAlgorithm 1 , bottom zoom level input resolution 5000 ×5000 17 . In following , present results measuring runtime required processthebottomlevelusingdiﬀerentnumbersofSparkexecutors ( Section5.1 ) .We also evaluate two resampling techniques GeoTrellis aﬀect run-time overall process discuss case ( Section 5.2 ) . Finally , discuss beneﬁts drawbacks using GeoTrellis ( Section 5.3 ) . 10 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Spark ClusterSSHFSGlusterFSInstance 1 Instance 16 Storage instanceInstance 15Spark Worker Spark WorkerSpark Executor Spark Driver ... Spark MasterSpark Executor Fig . 5.The setup Spark cluster . We use 15 executors GlusterFS distributed ﬁle system . 5.1 Scalability complete conversion process To test scalability , set Spark cluster ( see Figure 5 ) 15 executors . They running OpenStack [ 29 ] cluster Ubuntu 18.04as operating system , two CPU cores 8 GBof RAM . We set memory limit 4 GBfor Spark disabled swap avoid eﬀects measurements . The Spark master driver running togetheron separate instance . All input data stored additional instance andshared SSHFS mount [ 23 ] theexecutorsand thedriver . Theexecutorsthemselves stored shuﬄe data distributed ﬁle system GlusterFS [ 32 ] ( Version 5.6 ) spanned across instances . This means data tobe sent network time reading writing . We used setupfor two reasons : First , network connection much faster HDDaccess . This means additive network traﬃc aﬀect measurementstoo much . Second , amount shuﬄe data instance could changebased task distribution . By using distributed ﬁle system , ableto calculate required amount memory accurately avoid runningout disk space runtime . Tomeasurethedegreeofscalabilityofourapproach , weincreasedthenumber executors step step 1 15 triangulated complete test dataseteach time . We used bilinear resampling ( see Section 5.2 ) faster andgenerates smoother meshes . Afterwards , triangulated level 17 level 6.ThemeasurementresultsareshowninFigure6.Eachbarrepresentstherequiredprocessing time . This includes initial loading repartitioning sourcedata ( Setup step blue ) well triangulation diﬀerent levels ( parts bar ) . We compared runtime relation amount executors used . Given nrepresents total runtime tfornexecutors , calculated tn/nto get average runtime enfor nexecutors . In perfect setup linear 11 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 6.Total runtime based amount executors . The time level visualized diﬀerent colors . Time format hh : mm : ss Fig . 7.The blue line represents scaling factor approach . Lower values better . A factor 100 % means linear scaling . The purple line marks trend . scaling , t1would equal e1 , e2 , ... , e15 . We calculated scaling factor approach fn=en/t1and plotted results Figure 7 . The required runtime decreases almost linearly . In scenario lost ap- proximately ten percent scaling one worker 15 . This still substantial speed . As shown Figure 6 , bottom level ( red part bar ) requires time compared levels . This results high amountof points used input triangulation Algorithm 2 . For eachof points , distance current mesh calculated . This isexpensive many points . Later levels proﬁt reduction ofpoints ﬁrst step . A closer look system metrics shows CPU limiting factor triangulation leaf level 17 . It used nearly 100 % ( see Figure 8 ) , better CPU performance would especially improvethis step . The triangulation level 16 6 minor inﬂuence thetotal time CPU even fully utilized . This results increasedscheduling communication overhead many jobs processed fast . The green line represents Spark driver , contribute calculations . This CPU usage low triangulation . 12 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 8.CPU load processing data three instances . Fig . 9.Network usage processing data 15 executors . Inbound traﬃc time-axis , outbound traﬃc . The setup phase ( blue part Figure 6 ) includes loading repartitioning data well applying layout scheme . During loading phase whole dataset read disk . As mentioned , inputdata shared using SSHFS connection . You see data transmissionin peak blue line time axis Figure 9 . Afterwards , whenthe layout scheme applied , generated data written back disk.This time , data becomes distributed executors using GlusterFS . Asa result , see high network utilization ( inbound well outboundtraﬃc ) Figure 9 sharp increase disk usage Figure 10 . After period , original data longer used traﬃc triangulation caused Spark accessing shuﬄe ﬁles GlusterFS . It noted network connection bottleneck setup phase . The instances connected bandwidth 25 GBit/s , would allow much data transmitted . Instead , used HDDson instances limiting factor . They store generated datafast enough . An upgrade SSDs might result speed-up setupphase . In test , used memory limit 4 GBper executor . This high value allows Spark keep lot data memory . In Figure 11 , see thatmemory usage continuously grows point middle triangulating 13 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 10 . Disk usage processing data 15 executors . Fig . 11 . Memory usage processing data 15 executors . Fig . 12 . Memory usage processing data one executor . level 17 . When look one executor ( Figure 12 ) , limit reached already loading repartition phase . As Spark allowed toconsume memory conﬁgured , clean storage iteratively.This might result slowdown conﬁgurations less executors comparedtotheoneswithmorebecauseSparkcannotkeepdatainmemory.Theoretically , toanalyzethisinﬂuenceindetail , asingleexecutorwithunlimitedmemorywouldhave tested . 14 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.To summarize , conclude approach implementation suitable scalable processing large terrain data . As shown Figure 7 , theoverallruntimeoftheprocessscalesalmostlinearlywiththenumberofSparkexecutors . The bottlenecks described section connected availableresources ( main memory , CPU power , disk bandwidth ) well factthat GeoTrellis needs generate shuﬄe data . 5.2 Resampling techniques As discussed Section 4.1 , GeoTrellis resamples input data align layout scheme bottom zoom level . We inspected Bilinear NearestNeighbor ( NN ) resampling methods initial conversion step ( see Figure 3 , Step 1 ) inﬂuence runtime triangulation . The resulting meshes seen Figure 13 . For us , especially interesting comparethe conversion time deeper zoom levels , main bottleneck ofthe whole conversion process . The following results originate terraingeneration based single 5000 ×5000 pixels GeoTIFF . As mentioned , according Algorithm 1 , zoom level 17 suﬃciently ﬁne enough representour input data . However , following , compare even ﬁnerlevel 18 speciﬁcally demonstrate diﬀerences resampling methods . Comparing runtime ( see Figure 14 ) , one see NN bilinear sampling produce similar results level 17 . On zoom level 18 , conversiontimes diverge . NN much slower bilinear ﬁltering . However , adownside using bilinear sampling tile generation GeoTrellis : Theamount shuﬄe data Spark increases lot ( see Figure 15 ) . When converting whole dataset level 17 , total amount shuﬄe data bilinear sampling 440 GBcompared 165 GBwhen using NN , runtime still decreases . In case , implies bilinear sampling Level 17 Level 17 Level 18 Level 18 NearestNeighbor Bilinear NearestNeighbor Bilinear Fig . 13 . Generated terrain meshes zoom level 17 18 diﬀerent resampling methods 15 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.Fig . 14 . Conversion times resampling gridded height data . Fig . 15 . Total shuﬄe write depending chosen resampling techniques . superior , aslongastheSparkinstancehasenoughdiskspacetohandletheshuﬄe data . This counter-intuitive bilinear ﬁltering requires calculations toget interpolated value , NN needs ﬁnd closest point . When inspecting geometry generated meshes ( see Figure 13 ) , observe NN level 18 yields meshes containing geometries neitherpresent result bilinear sampling results conversions onzoom level 17 . This behavior due fact used dataset reso-lution approximatively matches pixel size sampled tiles zoomlevel 17 ( see Section 4.2 ) . Level 17 ﬁrst level without data loss . There-fore , NN sampling level 18 produces tiles contain pixel duplicates 16 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.the original raster dataset D. Because , triangulation needs time ﬁlter relevant points . This leads increased runtime much greater saving setup phase . Additionally , NN produces stair-likeeﬀect visible triangulated meshes Figure 13 . Bilinear sampling zoomlevel 18 removes pixel duplicates smoothing height information inthe tiles therefore yields similar results triangulations level 17 . This implies bilinear sampling preferred produces good results lower runtime compared NN sampling . 5.3 Beneﬁts drawbacks using GeoTrellis We used GeoTrellis provides lot functionality made develop- ment faster . Summing , provides following beneﬁts : – Loading GeoTIFFs . GeoTrellis load GeoTIFF ﬁles handle diﬀerent spatial reference systems . We need manage ﬁle accessesand directly use available data . – Applying layout scheme . For output ﬁles , comply speciﬁcation Tile Map Service ( TMS ) . For zoom level , thegenerated terrains aligned based speciﬁcation . GeoTrellisprovides easy way crop complete dataset required parts . – Integration Apache Spark . We want compute output distributed environment . Because strong integration GeoTrelliswith Apache Spark , need spend additional eﬀort parallelexecution multiple instances . Nevertheless , GeoTrellis drawbacks : – Usage raster data . GeoTrellis focused raster data . Whenever operationsaredone , newrastersarecreated.Thisbehaviorcancorrelatewitha loss precision . The results became visible Section 5.2 Figure 13 ( third column ) . Whenever required data granularity matchwith input data , resampling required . This lead wrong outputsdepending resampling method . – No control processing steps . GeoTrellisprocessesthelevelsonebyone . This leads lot shuﬄe data , calculation results one levelare required next one . Until point reached processingpipeline , data stored disk . A custom implementation couldprocess less detailed levels soon required subtiles exist . This wouldreduce amount shuﬄe data . – No control job distribution . Apache Spark handles distribu- tion jobs background . Especially input data stored adistributed ﬁle system , calculation speed could increased tiles areprocessed instance stored . In case , data doesnot need transmitted network . To summarize , GeoTrellis good tool get quick results . It provides lot functionality easily set . However sets limits possibleimprovements restricted raster data . 17 20 AGILE : GIScience Series , 1 , 2020 . Full paper Proceedings 23rd AGILE Conference Geographic Information Science , 2020 . Editors : Panagiotis Partsinevelos , Phaedon Kyriakidis , Marinos Kavouras This contribution underwent peer review based full paper submission . https : //doi.org/10.5194/agile-giss-1-10-2020 | © Authors 2020 . CC BY 4.0 License.6 Conclusions Future Work Our focus paper develop scalable approach create hierar- chical level-of-detail data structure optimized web-based visualization . Themain contribution approach distribute processing across cloud in-frastructure leverage available resources scale almost linearly . In orderto achieve , analyzed target format mapped data pro-cessing structure Apache Spark framework . This way , could parallelizethe triangulation splitting input data smaller tiles processingthem individually . The parallelization managed automatically Spark anddistributed so-called executors . This data-driven division processing stepsinto deployable standalone jobs enables scalability system regardto amount data . Based one stable eﬃcient conﬁguration , tested system terrain model dataset consisting 84 GBof GeoTIFF ﬁles . We several runs using dataset , incrementally increasing utilized executors . Figure 6visuallycomparestheresultingruntimesandprovesthatoursystemiscapableofalmost linearly reducing runtime regard utilized cloud resources . Our evaluation reveals initial setup phase Spark well CPUusageduringtriangulationforthebottomlevelofthehierarchyleavesroomfor improvements . In future work , investigate removal storageinstanceandinsteadusethediskspaceoftheexecutororadistributedﬁlesystemtoreducetheI/Oandnetworkoverheadatthebeginningofprocessing.Toreducethe CPU usage creating TINs , look algorithms creatingerror-bound TINs simpliﬁcation algorithms using quadric error metrics . \n"
          ]
        }
      ],
      "source": [
        "selected_strings = {}\n",
        "abstracts=[]\n",
        "conclusions=[]\n",
        "for i in range(1,10):\n",
        "  try:\n",
        "    vals = getACA(\"/content/\"+str(i)+\".pdf\")\n",
        "    abstracts.append(vals[0])\n",
        "    conclusions.append(vals[1])\n",
        "    selected_strings[\"/content/\"+str(i)+\".pdf\"] = vals[0] + ' ' + vals[1]\n",
        "  except:\n",
        "    continue\n",
        "\n",
        "  #author.append(vals[2])\n",
        "print(abstracts[0])\n",
        "print(conclusions[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NC7A4Ns3-dBL",
        "outputId": "09c3f7b5-77a9-4ef7-de69-124c64be652f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.8/dist-packages (3.6.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.8/dist-packages (from gensim) (1.15.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.8/dist-packages (from gensim) (1.10.1)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.8/dist-packages (from gensim) (6.3.0)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.8/dist-packages (from gensim) (1.22.4)\n"
          ]
        }
      ],
      "source": [
        "!pip install gensim"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tZG4tXssA_ho"
      },
      "outputs": [],
      "source": [
        "nltk.download('all')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Ehw1Z3HS-hl",
        "outputId": "92e1c668-00b7-4fa9-8247-39f0300ce184"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:gensim.models.base_any2vec:consider setting layer size to a multiple of 4 for greater performance\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The similarity of papers in descending order or similarity is:\n",
            "/content/3.pdf : 0.9889406958671666\n",
            "/content/5.pdf : 0.9602467547508797\n",
            "/content/4.pdf : 0.9593432387771939\n"
          ]
        }
      ],
      "source": [
        "import gensim\n",
        "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
        "from nltk.tokenize import word_tokenize\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "import nltk\n",
        "\n",
        "model = gensim.models.doc2vec.Doc2Vec(vector_size=30, min_count=2, epochs=80)\n",
        "tagged_data = [TaggedDocument(words=word_tokenize(_d.lower()), tags=[str(i)]) for i, _d in enumerate(main_string)]\n",
        "model.build_vocab(tagged_data)\n",
        "model.train(tagged_data, total_examples=model.corpus_count, epochs=80)\n",
        "\n",
        "data = word_tokenize(main_string)\n",
        "main_vector = model.infer_vector(data)\n",
        "\n",
        "selected_vectors = {}\n",
        "for i in selected_strings.keys():\n",
        "  data = word_tokenize(selected_strings[i])\n",
        "  selected_vectors[i] = model.infer_vector(data)\n",
        "\n",
        "def cosine_sim(a, b):\n",
        "  sum = 0\n",
        "  sum_a = 0\n",
        "  sum_b = 0\n",
        "  for i in range(len(a)):\n",
        "    sum_a += a[i]**2\n",
        "    sum_b += b[i]**2\n",
        "    sum += (a[i]*b[i])\n",
        "\n",
        "  return sum/((sum_a**(1/2))*(sum_b**(1/2)))\n",
        "\n",
        "cos_vals = {}\n",
        "for i in selected_vectors.keys():\n",
        "  cos_vals[i] = cosine_sim(main_vector, selected_vectors[i])\n",
        "\n",
        "print(\"The similarity of papers in descending order of similarity is:\")\n",
        "ranked = sorted(cos_vals, key=lambda x:cos_vals[x], reverse=True)\n",
        "for i in ranked:\n",
        "  print(i,':', cos_vals[i])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}