_buildDict.py

#!/usr/bin/env python3

import sqlite3, sys, re, os, zlib

#WORDLIST="data/wordlist.js"
#DICTLIST="data/dictionaries.js"
#SYLLABLELIST="data/syllablelist.js"
DB="webapp/TibetanDictionary.db"
DB_PRIVATE="webapp/TibetanDictionary_private.db"
CSV_INPUT="_input/dictionaries/public"
CSV_INPUT_EN="_input/dictionaries/public_en"
CSV_INPUT_PRIVATE="_input/dictionaries/private"
CSV_INPUT_PRIVATE_EN="_input/dictionaries/private_en"

# 32KB "dictionary" with frequent words to optimize deflate compression 
DEFLATE_DICT = " perfections pupil ru. skom Sumeru summary treat tshi vidyadhara voidness (1998) adorned battle bright, "+ \
"byung} clouds coemergent confused, deal Def.: dull fell formed got hairs ham learning, leg (lha life. "+ \
"limits (lit. MG: now, preliminary purity, relaxed rgyags {rta smell, state; statue tail thirty-two thunder "+ \
"useful abides achieve active alms bkas brel Chokgyur concentrations, dang/sa dang/yum direction, drags "+ \
"'dra'i {dri 'dris eighth form; 'gel go'i gzar gzhug ideas ill lhasa link market mouse mule Nepal occurring "+ \
"offering, {pha pit protective release shing. snyom spus symbol systems turn, unclear 2003).\\nCausative "+ \
"6th acceptable afflictive arising, Avalokiteśvara banner brdar bzhi}, chapter {C}not contradiction /de "+ \
"de, d+harma dpe'i ego forward garland gift, identical i.e. indicating instead Jewels kindness, meditating "+ \
"mis misery mix mtsho} mus nagas {nang nature. [not plan, proponent {ri rituals rul shake, shal speak, "+ \
"spirits, store thatness told tshag unchanging utter waste zhogs 6. artificial bshad/ chi chos} Clear "+ \
"continue correct, degeneration doors east eliminate enjoy fight formations gained grva hells hollow h+phu "+ \
"indicate instance ke Literally, meditation. mirror months \\nImperative: nyid, planet po}, political rtsad "+ \
"sambhogakaya {shing shong small, smod stop, su'u teacher, text, \"to tree. {tshogs {tshul worship yod, "+ \
"11th accumulated armor awareness; ba! beer blta/ bri bskyal bsngo bya} compositional conceive continuously "+ \
"creative custom designation develop developing harmony hi “I” journey khel light; {log mkhyend/ non-virtuous "+ \
"Panchen pang person. phenomena; possession regret rting scrutiny shangs shoulder shrine shug Songtsen "+ \
"spa synonymous tathāgata. Thousand transformation trust voice, about; Antiqua applications basket bcar "+ \
"center, chems comprise dang/bod {dga' dwell exchange fabulous guide, hard, Heart khor khrel 'khyog less "+ \
"lta/ meaning; needs nga'i non-existence pa: sage sbrags skas Solitary split, succession superior, thought; "+ \
"autumn base, birds bse dancing 'dar Drung esoteric eye, favorable Glorious grasped grum harmful heap "+ \
"hold, instruction, kyis} learned, loud mdung mentioned message methods or, person's phase phebs/dgung "+ \
"'phrad plan quality, sets {sgrub shad {shin skrangs slightly subjects sūtras tantra. Ten uncommon urine "+ \
"Vajrayana (= Also, bird, bones 'ching control, dharma, died dig etc., fragrant genuine, {gser help, )kyi "+ \
"lde'u others. paradises partial perfect, relation saddle servant, skyar sleeping stones unpleasant (10) "+ \
"affliction Anu astrological basis, blissful bsher burn check dang/2 dispute doctrinal emanations exhaustion "+ \
"gnas, gol gross hril in/ inferential joining leading [like mA Mahāyāna me, momentary non-existent performance "+ \
"performing presence, punishment receive, recite renowned session sogs. spras sri thoughts, translators "+ \
"visual wife, zhun 7th appeared be) beyond, boundless containing (dbu delusion detail dharmas, discriminative "+ \
"edge fat fold 'gags gates 'grogs groups gyo idle ka. kri (ma middle, mkhyen/ mnyan myu ngo/_ nirvāṇa "+ \
"novice opportunity promise provisions ra, reasons sgrigs shravaka stabilization states, therefore to. "+ \
"variety wander When why 32 Abbr. Avalokiteshvara beings; bkar byug chings cig. dbyig diligence, {dran "+ \
"'gyod hence however impermanence livelihood non-Buddhist nyam pa/5 'phur practicing praise, {rin sense, "+ \
"Shri stars TC.\\nPast: tibet, Tibetica treasures tsampa Vairochana 2003).\\nResultative AR brilliant carrying "+ \
"chu, come, cut, {dam (deng d+hi doubt, Dpa'-ris. exert {'gog gum lam/} literature lo} loving mammal. "+ \
"medium {mig mouth, nas} opening ox pig pur realms, relatives render rga Root suffer tenth arise, attribute "+ \
"bodhichitta Brahma chur conducive cultivate cure defilement degree dependence 'dor drawn embodiment excellent, "+ \
"Fifth {gang gdugs grog grown gsol/dgung g.yogs ignorance, individuals khyed kill lam, land, level, ornament, "+ \
"plant, purpose, remember (rgyal {rten saffron sgrom Soundings stupa the] throughout anger, around, btsal "+ \
"concentration, construction distinction electric evidence expression, feelings gnas} 'grul gseb gyin "+ \
"king's lpags mdzub meanings obstruction (or performed perhaps {phyogs (rang realization, rest, rje, rti "+ \
"Sandberg, scholar simply skull sons Stein, strong, student zhon apprehended beneficial building 'byar "+ \
"children dang/rgyal dbyin defilements drawing DS.\\nImperative: eightfold expel, fierce gsos hair, harvest "+ \
"kha, ljon means, mistake mkhal Mt. mthing na. Nagarjuna north raise, rdo'i reflection regarding {rgya "+ \
"siddhi solid sudden syn youthful bltas/ 'brom cing, decide devoted extent flowing fly {grub [h]; ([IW] "+ \
"karma, kayas la. late lud Mañjuśrī natures \\nFuture: omniscience perceived pointed promise, properly "+ \
"protectors receiving relationship ritual. sixth Sometimes Supreme tears treasury w, zhig] asked blind "+ \
"bo/ (bod 'brum doctrines English extraordinary getting {gsang gshen gtsigs investigation (Ja) kra'u ltag "+ \
"manifestations Mantra me'i mngal {mthong myed newly nun observed occur pa); person; pieces, powder pride, "+ \
"sba sun, \"the tshegs tsher verses warm 5. apprehension attendant collect, completely, concern contaminants "+ \
"created definitely dried everywhere gyong leads lnga, mass (mdo mentally (mi {nyan radiant remove round, "+ \
"scarf smras sta superficial together; traditional Tucci, viewpoint 23 24 bare blug boat broad conch concrete "+ \
"daughter dbyer gzengs (honorific) Hundred indirect issue maintain planets power/ rdzi remains {sdug sku} "+ \
"tense through, 'thus tshabs 17 awakened bent Buddha; Buddhas chinese danger DS.\\nMeaning: elder entered "+ \
"file fingers (for Germano, gnang/dgung {gzhi homage jewels make, mdza' obtained presentation rising sections "+ \
"skill thinking, {ting town wisdom. zer/de zhum {a bcol Body boiled btul by, bya/ chad/ collect craving "+ \
"dang/chos dbyi dimension distraction dvags else energy, freely harsh india jus Klong-chen-pa kyin manner, "+ \
"{nam {nges phugs Prasangika rdol resulting rgyal} rngul saying sees ska sked tiger volume yer alias buddhas. "+ \
"bun combined d+ha drive {'du gab goes Heaven Lake literal past. phor plain po}. principle quite rngog "+ \
"rtsi'i ruled spirit, strive sūtra. understand, unique visible wearing welfare yod/de (Chang chen, combining "+ \
"community destroyed exhausted gentle gradual history Khyentse {lta nomad occurs patience School service, "+ \
"status transmitted Tsang wish, zheng {zhi agreement bind brtsis 'bu color, defined frame friend, increasing "+ \
"interdependent Kadampa ma}, mdo'i mind/ mostly mtshor named noise perceive results samādhiḥ substance, "+ \
"thesis tied tradition. advice, age, bai btson business case, circumstances dbul enemies forces 'gos hear "+ \
"Karma khol lacking Liberation ltas pa/4 particle, Poetic presented pus question rdung rendered rgya'i "+ \
"threefold time; tsad want Wisdom, [with woman, 11) bend bkol blessings bo} brje brjid bzlas cag deer "+ \
"discussion dub examination expressed fixation grove gyes g.yul instant 'jim 'ju ldang lines lowest mindfulness, "+ \
"much, place/ po; powers, right, rlangs [ro Same {sgra sign, sort spelling surrounding Tantra. taste, "+ \
"zhor 15 5th (9) attention, B. discipline, drinking engagement events 'gyed 'gyurd/ happiness, happy, "+ \
"illusion khrom )la move, mthud {ngo old, phyung Samye scattered shes, situation, steps throat tshwa unity "+ \
"weak, arouse 'cham clinging [comp crooked defiled Definition: govt gyurd/ hill khru kind, logic press "+ \
"Pure rengs rmang (rnam sameness sde} setting skyur slog smaller wound zhing, 1971: bkrongs concealed "+ \
"concept described divide equality equipoise Essence firm, formative goods gshog implement kingdom {mtshan "+ \
"naga northern preparation rmi Sadhana skilled stir system, tshud visualization [w World already 'bri/ "+ \
"bsgoms bshig bsregs Commentary dakini grib {gsung impossible 'khrid mantras mchin 'ol past, perceiver "+ \
"purify recognize rkyal Samantabhadra saṅgha Section )shog sounds spar teaching, themselves victory wandering "+ \
".. ba]; brothers byes capable horn knowledge; 'on opposed philosophy probably room sbal square start "+ \
"stop TM travel wishing applied beginning, 'bral cave consciousnesses cry {dus forth; gather, grounds "+ \
"like, limitless 'phangs referring reply resultant rich robe sea Seven sred statement stretch thags 'thob "+ \
"worlds bram brgyan bslangs bsu confusion dgyes 'dom 'dri du} gem honor India. instrument listed movement "+ \
"na'i {nye object; rank strike style suitable, trees 'ur [1 apart &c.) correctly cultivation debate dharmata "+ \
"discriminating dualistic easily eastern give, initiation limit ljags many, Meru pair phod shing, skyes, "+ \
"spelled stable 1s alone, army {bde blessing blta bra 'brang bsnan bsrung btung cloud dharmin ger gson "+ \
"gzab Kagyü khrus {lhag ngas nying perfected phabs phogs po] practice. real, situation {snying strength, "+ \
"transgressions 'tshal using (with although (CH count cup data difference earth, faculties, knows liberation. "+ \
"might mngar mur Noble obstructions previously reverse robes ske spyang teachers above, addition answer "+ \
"caste {C}the g.ya' gzi hope lie little, loose Mahamudra Monastery nāma network neutral pour {sku smyug "+ \
"srab 'thor zog arms blazing brgyad, certainty compassionate corresponding dgun {dngos examine, 'gor Knowledge "+ \
"lci los marks. nāga {nyon oneself, price selflessness senses series sky, specifically sran understood "+ \
"another, appropriate attachment, Brahmā chin course crystal 'dra/ extrasensory faith, fifth for; gcung "+ \
"general, gtum lord, member Norbu, path. race rams surface temporary Way {yul 1, abodes acting (archaic) "+ \
"are: arrange {bya clan concepts ever ff. klu'i lhod ni/1 nirmanakaya origination practice; quick River "+ \
"rtsa'i search SRZT summit touch walk (1 better bodies brtul burden concordant diseases dpang glegs grwar "+ \
"gu'i g.yung include Jackson. killing matter, memory naked open, padma power; province ran reason, sick "+ \
"skyi southern standing subsidiary understanding, walking add at, beat brlabs bskrun built characterized "+ \
"depend ex. fruits intelligence lhas \\nMeaning: offer omniscient Prescriptions rough sbyard/ sbyord/ settle "+ \
"skrag snon sring suddenly sutras Thought uninterrupted wise brkos btus conduct, continuity covering destroy, "+ \
"extremely independent {'jigs ldogs/ mam musical non- OR pa/2 Realm remainder science spong/ subject, "+ \
"tr activity, before, bell break, byings compassion, divination empowerments family, followed gathered "+ \
"goms 'gyur/ heavy lab ldog/ overcome picture sga spang/ spangs/ spongs/ a] aware )bcas bros bsgrags bstsal "+ \
"{dmigs features free, generation gsog limited magic pull roof ru'i sbrang sha'i sickness split tell transitory "+ \
"ang bzhon chen} co drop g.yeng joined mchan million occasion protect rid sil 'thag Tibet, took Two western "+ \
"zang b+ha Bon bsdur categories consider day, dbar demons discursive disease, dpyod/} foundation harm, "+ \
"intent ki literally mo/ profit times, translator true, unobstructed valley weight A. advice bound clan. "+ \
"emptiness. followers hand, heard idea leaves mdzad/de observation party pillar sag sangha seek support, "+ \
"us winter {zag zi {'bras bsres class, Eighteen fear, 'khri measure, of. renunciation sent skong thread "+ \
"topics try well, bad, bamboo byas/ causing 'dzul eleven enjoyment error health heaven lady Manjushri "+ \
"(one pa]. rainbow samsaric sbyor/ scriptural several transcendent tshe'i utterly wear wisdom; Yisun. "+ \
"yung zlos 14 be, birth, bottom bshags can. commit dress gyur/ hu lifetime mastery morning rje} sbyar/ "+ \
"tsi Abhidharma {blo disturbing face, gsher gzhu la/ private {rdzogs rgas snyed step teachings. value "+ \
"ascetic Å  bowl byed} dbyug dust enlightenment. existence; firm gods, 'grub/ itself, kept kya nature; "+ \
"proof sla story treatises upon, {yang actions, angry deity, downfall dream generally (ka) perfectly rno "+ \
"“the 'u 20 blame, box burning 'jug} ma'am Mdo pa/3 poison progress rog sgril sing snar syn: weak which, "+ \
"yin. [1) bright create {dbu gces lineage, mahayana mdzo mgyogs obscurations record rta'i rules sbrel "+ \
"self, syllable tantra, TMC year, ?? being; bsgrub/ 'dzin, fresh immeasurable joyful khengs residence "+ \
"Sbram sgrubs/ thim tip Treasury. arhat avoid {bla branch carried category Fourth Gser g.yag hanging join "+ \
"minister mo} pa), people, sher straight dang/de'i dpyid establishment {'khor Namdak. respect, sgrub/ "+ \
"'thobs/ thob,'thob/ Yeshe accomplish 'bor brother bsil BT byar destruction don/ glog gyang immediately "+ \
"knife load mudra phenomena. placed synonym 'thob/ vital zhing/de 2003).\\nMeaning: 21th abandonment blugs "+ \
"{chu colored (dga' [DS].\\nFuture: guardian himself {las lding mos/ prana rays remaining byed/ can/ covered "+ \
"fit from; His joy, klung rgyugs roll {rtsa say, spring string {tshad wealth, {bdag (Beyer bsngags emotions "+ \
"exists gate gsan/dgung home intellectual mutually office perception, raise rather sder smoke stan tent "+ \
"{theg zan 100 bodily brjed daily did down; founder good, head, incense khe khre 'khyer non-conceptual "+ \
"objects, pervasion prajna prince ro/_ Tantras word, zhan 1self (8) bcu} border continents deluded distinct "+ \
"divided example, existence. fortune gdul goal hang indian king, klad lion mineral rtser rub Trisong bskyar "+ \
"change, collected composed dkyus emptiness, glorious habitual hungry it, jun (kha) 'khrugs New pa! prayer "+ \
"pure, thob/ truth, abode aggregate body; came connection delight gift growing immediate increase, ldem "+ \
"peak 'phrod process queen request resolve rites road [Syn White county genuine gshis Indra interest Mahayoga "+ \
"monk, narrow passing ster yogic approach bgos bsgrubs destroy entire fill gnyid keeping {lus mkho (sa "+ \
"{skyes 'tshol wind, *, acceptance bdun} (bka' brtsi {bsdus bsgrubs/ calm drum effort, etc] fire, gsum, "+ \
"Ibid. loss lost obstacles samaya {snang teeth went {yid 8th channel experienced forest khams} (met says "+ \
"{skabs thin unit wide attaining away; {bka' clean cow do, 'dzin} eighty famous gave g.yang identified "+ \
"opposite {rgyu salt snake substances 'tshams whether arrive beings. CM: colors command Dharma, flour "+ \
"focus {me primordially sgra'i sham sitting sos takes yal zil body. corpse darkness equanimity flow 'grig "+ \
"hide lcog outside quickly rtsam separate, sit stay ste/1 TC.\\nMeaning: verse wishes ze address ba'o/_ "+ \
"[be] btags,brtags/ continuous dro gdags/ lcang Middle {rigs table Vol. zhugs/ apparent (as author clarity "+ \
"clay death, expanse including influence jewel mthon nA nadi necessary Path pieces range skur soon sra "+ \
"articles buddhahood bus cing/de consequence cover, examine impermanent leader mdangs motion responsibility "+ \
"teachings, Vinaya 10) 13 aa bcings {C}one dal die discrimination fixed liberated ma) mgar mutual negation "+ \
"obtain Sautrantika sbyang solitary thebs Wisdom bhavati dgu'i 'dogs/ don't flat guard kalpa lam} meditation, "+ \
"mug nyang nyen {'phags {stong Text throne train alone devotion filled find read rer resting {skye smell "+ \
"yu accordance acts afflictions bgrod Domey further grasp 'ja' 'khon phru po/} pot rice Secret sound, "+ \
"SW thogs/ to/ arises ceremony clothing contact indestructible kham {lam love, realm, Sakya Second toward "+ \
"wall womb {ye after, anger chibs detailed dwangs families 'khel machine objective place; please poor "+ \
"sgreng complete, equivalent 'gros ni/ servant [Skt] sublime turned virtues 'bad bstar Buddhism caused "+ \
"cent chn entering explain gzungs held hong knower mtha'i realm. spo zam distance felt finger heart, length "+ \
"lngar minds mkha'i non obscuration pan reading snum village vinaya vol. below bzod/ characteristic charge "+ \
"'da' gzer illness {kha ljang lunar perceptions philosophical ripening seems serve victorious zhim charnel "+ \
"condition, follower food, Kaschewsky2. lham mchu med, ngor others, protection sgar yet 4th accept actually "+ \
"blos bsdad garment {gzhan kleshas KP1 lta'i medical name, scriptures {so song/ ta'i thab tsung brgal "+ \
"capacity cross flowers 'gul ldan, Maitreya mountains sūtra Third zlum 25 abandoning accumulation antidote "+ \
"authentic bstand/ clearly dbon especially extremes gdangs guide inference kings masters Mind 'phral possessed "+ \
"scripture sharp side, sreg stond/ behavior buddha. {'jig pa,, positive rmongs secondary wooden 11 air "+ \
"appearance, chig copper experiences Lag-len Mahayana po'am Precious re'i sheep te, view, zhung bltas "+ \
"happy holder lcam lte mistaken rdor {sangs sixteen bow easy eat eating entry greater 'khur Lingpa (mngon "+ \
"ngu'i pA paper worthy yag blang chus consort (deb earlier local neck [on page 'phang spread, They works "+ \
"yin/de zhol abide apprehending bzhi, consciousness; dod friends key limbs protector {rab sweet depending "+ \
"'dis Dzogchen gting looking official primary so, ston/ writing breath bsags bu, DK.\\nImperative: elephant "+ \
"go, house, in; money mtshos need {phyag still symbolic (1986) analysis can} cause, falling kindness KP3 "+ \
"KP4 (met) mkhris praise public rdzu realized {rig Six accomplishment brtse conceptions context dhyana "+ \
"establishing gzim inferior mthil thick tra 'tshe work, bodhicitta buddha, bzlog experience, gcan hat "+ \
"illusory purification rtse'i speaking tax twenty arrow brtsigs bsko bstan/ etc.] fact gor III ma/ master, "+ \
"mthu na/ persons (rgya thus fields meet myur sad zos accord considered dma' drin e.g. gathering Lotus "+ \
"LW meeting 'phan [R]; {rjes shri thing, appearing bone confidence ga'i intrinsic posture present, seen "+ \
"size spros TJ universe war capital force, green mna' palace possible really wisdom, 2, ground, gzung "+ \
"law mo, roots shed skyo spun bkag bro cutting exertion gzhal khron 'khrungs/dgung mundane sbug (A) broken "+ \
"cognition, devoid engage grow gtod Mount suffering, 'ba' [C] 'dabs dben nar truths unable usually yak "+ \
"zug belief figure 'joms la} lar lwa nose {rdo [as bskul cu'i directions 'doms dong fault First (Gyurme "+ \
"Kongtrül: la, 'pho retinue Śākyamuni skyor tsan wool 367 account along Chinese founded mind. {mngon mun "+ \
"part, shan abandon bstun forth, from, out; pa'o stand and, bases bsgom completion confused distant ex "+ \
"gan inside rgyur snye sutra tradition, yug becoming comes ling modes names rjod/ behind dmyal dra Kuijp "+ \
"purpose retreat accomplished bu} factor liberation, metal pa'ang path, snam tibetan bi cognitive definite "+ \
"glu is, notions ordination property 'dre [from lose producing proper showing tsho'i with; (7) circle "+ \
"conditioned cycle extensive gnya' {lha Light makes pa'o. shape yis 'degs je man, manifestation piece "+ \
"related royal schools star totally DK.\\nFuture: entity explained 'gro'i gtugs luminous ordained passed "+ \
"'phrog pleasure tha'i arisen fish 'khrab love mon phrad tongue area century Def. do/_ gather point, talk, "+ \
"teach aggregates cotton foot 'god merely mo. rdzun summer tse 2nd bos brjod/ essence, hearing imputed "+ \
"logs pleasant whom clear, display end, lesser logical mkhan/ peace po/ rdog rely stability (?) becomes "+ \
"listen moral sprad sprin dgos/ lkog reference snyigs stability. chud 'jags no/_ oil sgrog shun Vehicle "+ \
"zhig. 3rd am faculties insight substance abbr. barley bskyed/ dpa'i skyed/ bcud bodhisattvas direction "+ \
"na, ra'i zha // 4. function gone hole image mngags offered position realize recollection shel words, "+ \
"world, development ldum 'then ba'ang Buddha, cover fear MTTP. Purbuchog rtsub silver spirit tree, 28 "+ \
"dance few 'gran )lcags links often Padmasambhava refuge sbas sgo'i sleep Yoga coarse concerning 'dam "+ \
"gla light, mer practitioner sake skillful smug 'thung turning YTTM All bardo dharmakaya gyen med} 'phro "+ \
"phyi'i rope rtsed snow attached {de division KYT, Lama parts run TC.\\nCharacteristic TC.\\nNote: Yongdzin "+ \
"appearances dead flesh on; one, seal stick thor tshags against apply based rdzogs/ refer sgro skam space, "+ \
"towards treasure voice [among follow ho moving nyal pa/) points Sources. (5) awareness, bre gses, hell "+ \
"pa]; rogs suchness (Tournadre all, drink production receive rock action, application ba}, ba}. continuum "+ \
"lamp milk rlabs spro until 'bur Dung-dkar 'dzoms gshin Lhasa mgrin pu SK bton Dalai dbugs desire, lhung "+ \
"of/ brought klong lake nyid} Perfection TC.\\nIntransitive: 2003: days enlightenment, gad gdon ignorance "+ \
"reality, yol bran government gsod into, [to back, base 'byams krung men drod learning though vehicles "+ \
"bab enemy friend gain kan rise thought, torma 18 bzhir cog demon khug myang 'phar recitation thinking "+ \
"throw bskyangs definition freedom lto permanent she skyob study There added infinite nags origin sexual "+ \
"shes} wang ba; clothes dry innate once views always coming definitive doing identity impure separate "+ \
"deeds {'dod dud Eight help purified rain rdzob return worn 16 afflicted bzhad dwelling emanation golden "+ \
"lead leb luminosity of) play seat shang attitude could dark extreme hands letters ltos military myi see: "+ \
"vi. vow wife (chos gzung/ list rest sho thorough verb belonging explanation father feet included khyung "+ \
"TC.\\nTransitive: thu attention bzung/ )dang {don [for phenomena, thod absorption aspiration bskyod cyclic "+ \
"pain reach [TRS zungs/ 'a 'dab gsham India rgyug spyi'i stream translation about, bear call character "+ \
"continent grass gya gza' her khyon nyid/ things, tshom as, celestial contradictory existing {gzugs leather "+ \
"sadhana say seed Tibet. abandoned bsdoms carry ngar places rag sdang assembly bed best {dbang peaceful "+ \
"received Rinpoche sger tig way, As knowledge, soft (thoroughly) vast water, -- consciousness, existent "+ \
"so. taste thengs thur ability CH meat spyir crown gzan harm pe revealed rkyang spontaneous wish 'bangs "+ \
"bskos 'chang half realizing substantial ban connected power, produce too Utpal doubt following live note "+ \
"Nyingma powerful [R]. [syn: thousand 9) contemplation dbyar dpyod everything ldebs sde'i shos 2003).\\nCharacteristic "+ \
"butter dril phrug (/ {bsam ca causal gas g.yu internal original pi {rgyal sometimes spos country meaning, "+ \
"specific sphere young (a found gra rdza rgyang {shes zing beautiful bsal dur lags Nomads {sa section "+ \
"service since commentary ornaments 'thab whole animals buddhist hundred pad+ma valid beings, faculty "+ \
"'grus ming. paN+Di sdud suitable temple associated brtags door 'dzin/ mgron mixed big khung next strength "+ \
"bca' due mdor sensory strong years bam language mean mtshar title (2001) appear attributes 'bebs establish "+ \
"learned skra ask break (h) 'Jig-rten-mgon-po, reasoning sgor tsang Buddha. [by dza texts vision bcu'i "+ \
"city 'grub possessing rtsig species truly dan front {gnas leave must relative verbal bskur directly hidden "+ \
"Obl.] phyis sdig absolute According inherent levels nyag over, state, DD 'dzam ear exist feast method "+ \
"'phen skud terma then yangs fourth Jamgön pride spontaneously zhwa Bka'-'bum brtsegs bsad Compare group "+ \
"'khyil practices sgrol think written gzhag off, silk arise around 'gag gral 'jal Lord mal ne show book "+ \
"lama matter our center (e.g.: far gtad holding that, 'chi guru round endowed feeling fine mere )nang "+ \
"{byang cattle de} ngal ocean equal order, sngo These wa 2003).\\nTransitive: blue child conditions hot "+ \
"understand appears kha'i ND.\\nMeaning: bsgrub existence, itself ways auspicious divisions up; 2003).\\nIntransitive: "+ \
"intention produced ching g.yog magical skya'i gzhir [Abs. deep ND. /} fundamental pristine vase byis "+ \
"klog now 'o ornament phog phud attachment control draw low ngang rtsol {dge either gnyis} night case "+ \
"cast le'u remain lack spirits zho bu. characteristics gal res Sanskrit Bodhisattva done dum Karmay, samadhi "+ \
"branches intermediate lon lu object, 'os skyel (6) became bkur mind; particle perform tibet region rgol "+ \
"Vajra dogs every instruction 'khrug mode mt disciples dor heat ones we 'byin dang/de LZ.\\nMeaning: Samdo "+ \
"superior taken conception gleng lives neither ril merit )sogs universal bzhed energy gnon life, Buddha's "+ \
"difficult ri'i skyabs BBNP gnyan khal gur line mdzad/dgung send tshangs false karmic lce rdul regard "+ \
"bsgrigs ngu signs union 'brog pags sgyur skrun (to cin )las offerings sky GD khur mouth would Abs.] enter "+ \
"shor snyad spyad ??? KYN, absence holy sku'i taught subsequent tshur age Buddha’s 'dogs gyal 'i precepts "+ \
"tantras classes mark again DG lha'i moment pay gna' material effect khyod spyad/ (the deb monks spyod/ "+ \
"terms tshon being, byed, dman negative conventional Chödag. manner rebirth yellow bong district marks "+ \
"nirvana cold grang mother stegs tea bsdu him increase (4) gus happiness nyis feel 'gram support eighteen "+ \
"faults pad stone dependent sel dang/ ther ge'i drug} oral tshar BP exalted manifest slebs person, )zhes "+ \
"animal activities conduct effort khrir sdong bzod forms illus. la'ang ma. bliss change [Erg. gyon (Tha "+ \
"buddhas instructions nine brnyan bud Dpe-chos Five Guru lnga} grasping quality away, mindfulness period "+ \
"shag beginning khar total together, move source Yogacara-Svatantrika-Madhyamika 1992: deng Gces for, "+ \
"ja joy letter third g.yon here langs rnga rwa tham zhen concentration example paths stages dbyings pham "+ \
"practice, element {ma personal purity speak Sutra eyes mandala speech, attained formless times early "+ \
"form, sprel central faith gnad khra wealth compassion flower my tshong grain gzhi'i giving presence brgyab "+ \
"expression going 'bar previous shAkya [= phab skin collection know powers refers tantric 'dres final "+ \
"hard goddess medicine zangs particular transmission hor Jetsunpa lotus stangs tsong bstod factors gri "+ \
"rnyed gives vessel care understanding 'char vows causes gsum} phenomenon actual discipline sgang lor "+ \
"phur dmigs/ gangs ku nothing snyam subtle dgag rong attain mes abiding TC.\\nVoluntary: bang virtue disciple "+ \
"pa'o/_ phyed sgron spu khog ngam result blood nyung open ram sgyu while chab 'khrul empowerment gold "+ \
"in, nan someone brgyad} han cha'i never zung similar {sems above ancient last gam nature, no. future "+ \
"kar lay nya skal essential attainment mor wild gro ltung mos profound For miraculous rtswa sacred Stein. "+ \
"mi'i new shul chu'i samsara iron 600 bslab 'phrin with, khrag sne spread tshos He mya yos divine land "+ \
"slar tshor ba] glo grogs let reg measure cannot near sbrul top phran TC.\\nInvoluntary: anything former "+ \
"phu tho cung 'gan mdzes states drang important sbyangs [future: {kun bird 'bring ko benefit snang/ worldly "+ \
"wrathful cessation middle left use highest zhag doctrine phyugs bskal dkon fruit talk (in much of: phug "+ \
"tsa monastic babs si spong activity color time, yun 1st cloth short thad gdong gnam plant principal various "+ \
"ka'i ljongs {mi vehicle 'chad ches lang can, number basic po. Name wood elements teacher male pa}, phal "+ \
"zhang rags outer rna seeing 8) knowing 'byung/ bdud byung/ making TD, des dmangs khro major naturally "+ \
"ba}; tshal King ral to; condition gtor perfection 12 excellent force g.yas mantra rgod SS thoughts close "+ \
"dwags khyi fall stage training tshab brtsams Dorje tsho deities horse preparation. sgrib 'khrungs sgrig "+ \
"khrod sprod aspect khral oneself primordial certain minor second down, yod/ mjal [ggd] later spangs gzhan/ "+ \
"real shod Tantra pod twelve dper pa; phar empty rma chief whose bum (3) mched shi look yin/ sgam sign "+ \
"thung 'dzugs system brtag fully subject n. place, lhun lo'i on, sdod ba) hair wheel should srang enlightened "+ \
"of; spel mdog shugs 'byed heart mtsho'i nu byad lhan p. pass text 'debs given off precious zab had {rnam "+ \
"woman 'bod ma, An chod char god khas 'tsho fruition paN act 'bab dharmas whatever bzhi} ma} reason [a "+ \
"birth during Three mjug g.yo mdun before stabs hand It bcug rtsod Tenpa bza' external gzhis skyid some "+ \
"were epithet school chang thoroughly keep moon eye srin wind dung ritual skyed intr. 'ong single phun "+ \
"yoga author) lord pa}. rje'i turn appearance Dargyey (Sera-mey tshugs myong family monastery chas earth "+ \
"thogs 3. conceptual reality bal bsten forth dog living trans. dug bcom 'khod of] rgan taking textbook "+ \
"TSE common grol khu spungs Skt. tshun longs CHN river dpag klu gtam virtuous brtson IV side type dpa' "+ \
"snyoms khong translated house mountain v.; face among bskor spyan these correct gtogs higher [mss] (ACIP: "+ \
"even [DS], rgyab bzos monk individual (tha gras TC.\\nSyntax: upper body, both upon 'gal physical sogs} "+ \
"well wrong blangs isc. 'gog rgyu'i 'bul gzhon order black sog special byams ordinary skyon spang each "+ \
"gnod most btsugs 'don khyer 'bru TC, yo cho high rgyag born herb zer/ past rtogs/ vajra yum &, may mda' "+ \
"cut them mtshungs po, ltos/ Four {chos (Hackett large 'dren chag hold se month tshig) bring btab phrag "+ \
"realms beyond son dka' realization able tshigs end mkhar kong so'i respect che'i byi dpung JD sam 7) "+ \
"level sang word mongs perfect thun 'di'i gsan people actions sentient thug religious thal dngul gdams "+ \
"10 ft. lnga'i smin disease how dpar etc different another gru ting khab arising space general mong smon "+ \
"bur ti Also dpyad established (of gses yar cang gsol (Hoshi field self dbyibs 1's said imp. 'go thig "+ \
"nyes byin essence ldog truth Buddhist Comment: class yong between ched your action jo thing tshes tsA "+ \
"bo'i death li lineage cher speech tradition bag mdzod rnal bodhisattva together bsngal human evil khungs "+ \
"pa}; old phra ba'am drangs back inner yas 'brug dgra Indian brda seven main point lde phyug snod 'ga' "+ \
"thought khrid known just day 'dul gros mtho nyon bshes phung basis master pa] tantra dran LZ. rtag 'bum "+ \
"year sbyin lower same bye does head phul sun (see complete offering drag gter nams than gnyen little "+ \
"because such within work lan full 'jog out, 'phrul le shas rtsal right srung ci kho 'phel term zag root "+ \
"bor nge noble teaching perception bsod seng skar ultimate za mind, more rus (T) so/_ 'jigs pa) phag shin "+ \
"nyan bral suffering bskyed 'dir tshod dkyil {rang srog gods if been pf. spiritual phye bsnyen lug experience "+ \
"ba/ supreme cognition female bcad direct rtsom brtan dbye secret karma sngar bzhi'i completely ras sgom "+ \
"sbyar khongs sbyong gnyer TC.\\nFuture: mtshon rtsi sound II po} O.T. ba/} ji set slong only rlung khyim "+ \
"objects enlightenment Example: 'jig This according bya'i byon over thos Great away liberation fire zhi "+ \
"zhus qualities ... zad medicinal gcod called zur sprul Tibet zhe ground nub TC.\\nImperative: food 'jam "+ \
"ste, smra gso red [the mas mu ten dharma dri sor natural desire grong part realm mgon clear down gung "+ \
"others etc. zas zhabs gdung words rkang bkra med. zhib aspects zin about mkha' pha dbyangs lho view ha "+ \
"bris gu bcos under thon bad world nyin bon 'du [in any light Blan to, dgongs tA yab get phenomena but "+ \
"mnga' skya ta bzhengs those btsun 6) something skul 'phags true blon emptiness shog sdom shu ro awareness "+ \
"stobs btags gshegs rkyen 'dus mgo rtsis sar sdug bzhes snyan deity dbu glang 'dun (2) bzhag white bzo "+ \
"of, up, come tog mthar (byed free (1) eight gos existence made where dgung types In theg gsung you [R] "+ \
"meditative nus consciousness ris shis many mkhyen bco dbus skyong tree w/o meditation Source: mtshams "+ \
"Dharma rol [one there brang 'grel khul mo'i teachings steng rnying zhugs stag zhal buddha rtags they "+ \
"stod log 'ongs Coll. Zhi-byed gtsug tan man bsgyur king 'og kinds rdzas cing sense present put khyab "+ \
"used tsha dpe nyer bu'i chub rdzong ser mnyam [past: pos tha [h] smad bden thabs srong ya nye small bkod "+ \
"yid rtog dmigs first rung gtong 'dug thugs mthong khyad after sa'i when life btang gtso snga lhag tshan "+ \
"gzigs phyin gser phan chog \\nPast: meaning [arch] chad 'byor dmar yan beings brten cu pure brjod sbyor "+ \
"thag thar knowledge drung mig become phyir will [of brag 'od rtogs nga gsungs give min 2. khrims mtsho "+ \
"zhu yongs la'i dgu rgyun person pa'am other sras own water thob ga khag shar gsang practice kind CD. "+ \
"nature things khams One gtan brgyud bzung cause their rdzogs rgyan ngos way tshe nam Tibetan snying pho "+ \
"ba. ngan rgyus song brgya out sha gtsang sna thub der means thams ru pa/ pa/} gzugs state nad mchod dar "+ \
"form nyams wisdom gdan DS. he ge cad gsar nag take grwa srol was sngags through yon cig nor dmag sman "+ \
"phreng good nges dgon legs phyag sgra ces Present: its lcags Buddha mdo mthun long skyes sgrub power "+ \
"bsam also skye no bzhugs por rta 'bri 'dod gzhi what ND, bde six sngon ngag very 5) len chags object "+ \
"sum dad ye gling shig mngon spyi mchog rtse phyi bas mkhas 'brel name lung place without up skor time "+ \
"do one's de'i 'dra gyur gar mental sgo body DK. 'bras dpal spyod having make like ra ngo snang stong "+ \
"phebs nyi da zla thang lus CD, bsdus mtha' sangs path mar gnang 'jug ba} 'das his grags tshang tshad "+ \
"grangs blo btsan 'gyur dgos have dga' skad 'byung 'gro rabs dam te/ two ston rim LZ, has gzhung bdag "+ \
"into yig re byang dkar this rin DK, chung being TC. mkhan dpon rtsa bdun khri kun kyang rgyu Syn dang, "+ \
"bka' ltar gong rten bzhin (PH) gis yi 1. gzhan tsam 'di ka pa. lugs sde ring ba, tshogs me srid dngos "+ \
"phyogs rdo bstan dag 'khor ri brgyad see rig zer grub bshad five mind bla bzang ste great rgyud 4) zhing "+ \
"tshul slob ma'i cha sems lag gang DS, gsal dge go drug SA See rjes rje at lnga it lam shing ldan rgyas "+ \
"nyid thog tshig mtshan bzhi mdzad skabs gyis chu khang yul so three all bo who po'i 'dzin dus mang four "+ \
"kyis bod te are rigs lha che kha pa} can byas an which bya ste/ bu sku bu/ mo bcas rab rgya rnams gcig "+ \
"To pa, bcu pas tu Btsan-lha. yang rnam shes one The dbang rang 3) ming on gsum nang with zhig from gnas "+ \
"not bar byung that na ni gnyis yin sa sogs don med for zhes by rgyal yod lo de or chos as su las byed "+ \
"be chen lta is mi 1) ba'i par 2) gi gyi po nas du ma dang in ba kyi la and to pa'i pa the of "

DEFLATE_DICT_BYTES = DEFLATE_DICT.encode("utf-8")

def deflate(text):
    textBytes = text.encode("utf-8")
    co=zlib.compressobj(9,zlib.DEFLATED,-15,9,zlib.Z_DEFAULT_STRATEGY, DEFLATE_DICT_BYTES) 
    data=co.compress(textBytes)
    data=data + co.flush()
    
    return data


def createDb(path):

    if os.path.exists(path):
        os.remove(path)    
    
    db = sqlite3.connect(path)
    db.execute("create table DICTNAMES(id int, language text, name text)")
    db.execute("create table DICT(term text, dictionary int, definition blob)")
    db.execute("create table DICT_EN(term text COLLATE NOCASE, dictionary int, definition blob)")
    db.execute("create table \"android_metadata\" (\"locale\" TEXT DEFAULT 'en_US')")
    
    return db


def cleanupTerm(value, isTibetan):
    value = value.replace("\""," ")
    value = value.replace("-"," ")
    value = re.sub(r"^\s+","",value)
    value = re.sub(r"\s+$","",value)
    value = value.replace("\\n"," ")
    value = re.sub(r"\s+"," ",value)
    value = value.replace("­","") # delete zero-width non-joiner

    if isTibetan:
        value = value.replace("v", "w")

    return value


def cleanupDef(value):
    value = value.replace("\"","\\\"")
    value = re.sub(r"^\s+","",value)
    value = re.sub(r"\s+$","",value)
    value = re.sub('"','\\"',value)
    value = value.rstrip("\r\n")
    value = value.rstrip("\r")
    value = value.rstrip("\n")
    
    return value

def getDictNameFromFile(dictFile):
    return re.sub("^.*[0-9][0-9]-","",dictFile)



def processFile(dictFile, db, dictNr, isTibetan):
    if isTibetan:
        language = "bo"
    else:
        language = "en"
        
    
    sqlQuery = 'insert into DICTNAMES values({0},"{1}","{2}");'.format(dictNr, language, getDictNameFromFile(dictFile))
    db.execute(sqlQuery)

    if isTibetan:
        tableName = "DICT"
    else:
        tableName = "DICT_EN"
        
    
    existingTerms = {}
    with open(dictFile, 'r') as inp:
        
        for line in inp:
            if ( not line.startswith("#") ) and ( "|" in line ):
                term, definition = line.split("|");
                
                term = cleanupTerm(term, isTibetan)
                definition = cleanupDef(definition)
                
                
                lineContents = term + "|" + definition
                if (term != "") and (definition != "") and (lineContents not in existingTerms):
                    # todo Check, if same entry was already seen before!
                    definitionBlob = deflate(definition)
                    sqlQuery = 'insert into {0} values("{1}",{2},X\'{3}\');'.format(tableName, term, dictNr, definitionBlob.hex())
                    db.execute(sqlQuery)
                    existingTerms[lineContents] = 1
                
                #tibetan = cleanupValueMinimal(tibetanOriginal)
                #englishTerms = cleanupValue(englishOriginal)
            

def processFolder(db, srcFolder, isTibetan):
    dictNr = 1
    for dictFile in os.listdir(srcFolder):
        dictFileWithPath = os.path.join(srcFolder, dictFile)
        if not os.path.isdir(dictFileWithPath):
            print(dictFileWithPath)
            
            
            processFile(dictFileWithPath, db, dictNr, isTibetan)
            db.commit()
            dictNr+=1
            

    db.commit()

def closeDb(db):
    db.execute("create index i1 on DICT(term, dictionary);")
    db.execute("create index i2 on DICT_EN(term COLLATE NOCASE, dictionary);")
    db.execute("create index i3 on DICTNAMES(language, name, id);")
    
    db.commit()
    db.close()
    


def main():
    print("=== PUBLIC DICTIONARIES ===")

    db = createDb(DB)
    
    print("- Processing Tibetan-English Dictionaries")
    processFolder(db, CSV_INPUT, True)
    
    print("- Processing English-Tibetan Dictionaries")
    processFolder(db, CSV_INPUT_EN, False)
    
    closeDb(db)


    if os.path.exists(CSV_INPUT_PRIVATE):
        print("=== PRIVATE DICTIONARIES ===")
        db = createDb(DB_PRIVATE)
    
        print("- Processing Tibetan-English Dictionaries")
        processFolder(db, CSV_INPUT_PRIVATE, True)
    
        print("- Processing English-Tibetan Dictionaries")
        processFolder(db, CSV_INPUT_PRIVATE_EN, False)
    
        closeDb(db)

main()