From 867d20d29f62f8f322c8624739689d37b626825c Mon Sep 17 00:00:00 2001 From: Louis Date: Sun, 9 Oct 2022 02:18:18 +0200 Subject: [PATCH] Fixed run() entrypoint Added debug files to .gitignore Added f-Strings in print statements Added Dockerfile Removed unnecessary packages from requirements.txt --- .gitignore | 5 + Dockerfile | 6 + README.md | 13 +- requirements.txt | 29 +--- src/getData.py | 4 +- src/main.py | 402 ++++++++++++++++++++++----------------------- src/processData.py | 34 +--- 7 files changed, 231 insertions(+), 262 deletions(-) create mode 100644 Dockerfile diff --git a/.gitignore b/.gitignore index 85728f7..ac8a59b 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,8 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Debug files +arnis-debug-raw_data.json +arnis-debug-processed_data.json +arnis-debug-map.png \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a47afd0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.9 +RUN apt-get update && apt-get -y install git ffmpeg libsm6 libxext6 +RUN cd /home && mkdir /home/region && git clone https://github.com/louis-e/arnis.git +WORKDIR /home/arnis +RUN pip install -r requirements.txt +ENTRYPOINT ["python", "arnis.py"] \ No newline at end of file diff --git a/README.md b/README.md index 546b067..5f1e9e5 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,14 @@ Notes: - The city, state and country name should be in the local language of the respective country. Otherwise the city might not be found. - You can optionally use the parameter ```--debug``` in order to see the processed values as a text output during runtime. +### Docker image +If you want to run this project in a container, you can use the Dockerfile provided in this repository. It will automatically scrape the latest source code. After running the container, you have to manually copy the generated region files from the container to the host machine in order to use them. When running the Docker image, set the ```--path``` parameter to ```/home```. An image on Dockerhub will follow soon. +``` +docker build -t arnis . +docker run arnis --city "Arnis" --state "Schleswig Holstein" --country "Deutschland" --path "/home" +docker cp CONTAINER_ID:/home/region DESTINATION_PATH +``` + ## :cd: Requirements - Python 3 - At least 8 Gigabyte RAM memory @@ -84,17 +92,18 @@ ID | Name | Note | 70-79 | House interior | The last digit refers to the building height | ## :memo: ToDo -- [ ] Add Dockerfile -- [ ] Use f-Strings in print statements - [ ] Implement multiprocessing in floodfill algorithm in order to boost CPU bound calculation performance - [ ] Add code comments - [ ] Implement elevation +- [ ] Find alternative for CV2 package - [ ] Improve RAM usage - [ ] Add interior to buildings - [ ] Optimize region file size - [ ] Street markings - [x] Automated Tests - [x] PEP8 +- [x] Use f-Strings in print statements +- [x] Add Dockerfile ## :copyright: License MIT License[^3] diff --git a/requirements.txt b/requirements.txt index 3843f3b..e81e46f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,32 +1,9 @@ anvil-parser==0.9.0 -argparse==1.4.0 -black==22.8.0 -certifi==2022.9.24 -charset-normalizer==2.0.12 -click==8.1.3 -cycler==0.11.0 -flake8==5.0.4 -fonttools==4.37.4 -frozendict==2.3.4 -idna==3.4 -kiwisolver==1.4.4 matplotlib==3.5.2 -mccabe==0.7.0 -mypy-extensions==0.4.3 -NBT==1.5.1 numpy==1.22.3 opencv-python==4.5.5.64 -packaging==21.3 -pathspec==0.10.1 -Pillow==9.2.0 -platformdirs==2.5.2 -pycodestyle==2.9.1 -pyflakes==2.5.0 -pyparsing==3.0.9 -python-dateutil==2.8.2 python-polylabel==0.6 requests==2.27.1 -six==1.16.0 -tomli==2.0.1 -typing_extensions==4.3.0 -urllib3==1.26.12 +argparse==1.4.0 +black==22.8.0 +flake8==5.0.4 \ No newline at end of file diff --git a/src/getData.py b/src/getData.py index c54dc94..5fd78d7 100644 --- a/src/getData.py +++ b/src/getData.py @@ -38,7 +38,7 @@ def getData(city, state, country, debug): """ ) - print("Chosen server: " + url) + print(f"Chosen server: {url}") try: data = requests.get(url, params={"data": query1}).json() @@ -51,7 +51,7 @@ def getData(city, state, country, debug): elif "Dispatcher_Client::request_read_and_idx::rate_limited" in str(e): print("Error! IP rate limited") else: - print("Error! " + str(e)) + print(f"Error! {e}") os._exit(1) if debug: diff --git a/src/main.py b/src/main.py index bf8cb94..c9af357 100644 --- a/src/main.py +++ b/src/main.py @@ -108,228 +108,218 @@ def saveRegion(region="all"): if region == "all": for key in regions: regions[key].save(mcWorldPath + "/region/" + key + ".mca") - print("Saved " + key) + print(f"Saved {key}") else: regions[region].save(mcWorldPath + "/region/" + region + ".mca") - print("Saved " + region) + print(f"Saved {region}") -rawdata = getData(args.city, args.state, args.country, args.debug) -imgarray = processData(rawdata, args) +def run(): + rawdata = getData(args.city, args.state, args.country, args.debug) + imgarray = processData(rawdata, args) + print("Generating minecraft world...") -print("Generating minecraft world...") - -x = 0 -z = 0 -doorIncrement = 0 -ElementIncr = 0 -ElementsLen = len(imgarray) -lastProgressPercentage = 0 -for i in imgarray: - progressPercentage = round(100 * (ElementIncr + 1) / ElementsLen) - if progressPercentage % 10 == 0 and progressPercentage != lastProgressPercentage: - print( - "Pixel " - + str(ElementIncr + 1) - + "/" - + str(ElementsLen) - + " (" - + str(progressPercentage) - + "%)" - ) - lastProgressPercentage = progressPercentage - + x = 0 z = 0 - for j in i: - setBlock(dirt, x, 0, z) - if j == 0: # Ground - setBlock(light_gray_concrete, x, 1, z) - elif j == 10: # Street - setBlock(black_concrete, x, 1, z) - setBlock(air, x, 2, z) - elif j == 11: # Footway - setBlock(gray_concrete, x, 1, z) - setBlock(air, x, 2, z) - elif j == 12: # Natural path - setBlock(cobblestone, x, 1, z) - elif j == 13: # Bridge - setBlock(light_gray_concrete, x, 2, z) - setBlock(light_gray_concrete, x - 1, 2, z - 1) - setBlock(light_gray_concrete, x + 1, 2, z - 1) - setBlock(light_gray_concrete, x + 1, 2, z + 1) - setBlock(light_gray_concrete, x - 1, 2, z + 1) - elif j == 14: # Railway - setBlock(iron_block, x, 2, z) - elif j == 20: # Parking - setBlock(gray_concrete, x, 1, z) - elif j == 21: # Fountain border - setBlock(light_gray_concrete, x, 2, z) - setBlock(white_concrete, x, 1, z) - elif j >= 22 and j <= 24: # Fence - if str(j)[-1] == "2": - setBlock(cobblestone_wall, x, 2, z) - else: - fillBlocks(cobblestone, x, 2, z, x, int(str(j[0])[-1]), z) + doorIncrement = 0 + ElementIncr = 0 + ElementsLen = len(imgarray) + lastProgressPercentage = 0 + for i in imgarray: + progressPercentage = round(100 * (ElementIncr + 1) / ElementsLen) + if ( + progressPercentage % 10 == 0 + and progressPercentage != lastProgressPercentage + ): + print(f"Pixel {ElementIncr + 1}/{ElementsLen} ({progressPercentage}%)") + lastProgressPercentage = progressPercentage - setBlock(grass_block, x, 1, z) - elif j == 30: # Meadow - setBlock(grass_block, x, 1, z) - randomChoice = randint(0, 2) - if randomChoice == 0 or randomChoice == 1: - setBlock(grass, x, 2, z) - elif j == 31: # Farmland - randomChoice = randint(0, 16) - if randomChoice == 0: - setBlock(water, x, 1, z) - else: - setBlock(farmland, x, 1, z) + z = 0 + for j in i: + setBlock(dirt, x, 0, z) + if j == 0: # Ground + setBlock(light_gray_concrete, x, 1, z) + elif j == 10: # Street + setBlock(black_concrete, x, 1, z) + setBlock(air, x, 2, z) + elif j == 11: # Footway + setBlock(gray_concrete, x, 1, z) + setBlock(air, x, 2, z) + elif j == 12: # Natural path + setBlock(cobblestone, x, 1, z) + elif j == 13: # Bridge + setBlock(light_gray_concrete, x, 2, z) + setBlock(light_gray_concrete, x - 1, 2, z - 1) + setBlock(light_gray_concrete, x + 1, 2, z - 1) + setBlock(light_gray_concrete, x + 1, 2, z + 1) + setBlock(light_gray_concrete, x - 1, 2, z + 1) + elif j == 14: # Railway + setBlock(iron_block, x, 2, z) + elif j == 20: # Parking + setBlock(gray_concrete, x, 1, z) + elif j == 21: # Fountain border + setBlock(light_gray_concrete, x, 2, z) + setBlock(white_concrete, x, 1, z) + elif j >= 22 and j <= 24: # Fence + if str(j)[-1] == "2": + setBlock(cobblestone_wall, x, 2, z) + else: + fillBlocks(cobblestone, x, 2, z, x, int(str(j[0])[-1]), z) + + setBlock(grass_block, x, 1, z) + elif j == 30: # Meadow + setBlock(grass_block, x, 1, z) randomChoice = randint(0, 2) + if randomChoice == 0 or randomChoice == 1: + setBlock(grass, x, 2, z) + elif j == 31: # Farmland + randomChoice = randint(0, 16) if randomChoice == 0: - setBlock(wheat, x, 2, z) - elif randomChoice == 1: - setBlock(carrots, x, 2, z) + setBlock(water, x, 1, z) else: - setBlock(potatoes, x, 2, z) - elif j == 32: # Forest - setBlock(grass_block, x, 1, z) - randomChoice = randint(0, 8) - if randomChoice >= 0 and randomChoice <= 5: - setBlock(grass, x, 2, z) - elif randomChoice == 6: - fillBlocks(log, x, 2, z, x, 8, z) - fillBlocks(leaves, x - 2, 5, z - 2, x + 2, 6, z + 2) - setBlock(air, x - 2, 6, z - 2) - setBlock(air, x - 2, 6, z + 2) - setBlock(air, x + 2, 6, z - 2) - setBlock(air, x + 2, 6, z + 2) - fillBlocks(leaves, x - 1, 7, z - 1, x + 1, 8, z + 1) - setBlock(air, x - 1, 8, z - 1) - setBlock(air, x - 1, 8, z + 1) - setBlock(air, x + 1, 8, z - 1) - setBlock(air, x + 1, 8, z + 1) - elif j == 33: # Cemetery - setBlock(podzol, x, 1, z) - randomChoice = randint(0, 100) - if randomChoice == 0: - setBlock(cobblestone, x - 1, 2, z) - setBlock(stone_brick_slab, x - 1, 3, z) - setBlock(stone_brick_slab, x, 2, z) - setBlock(stone_brick_slab, x + 1, 2, z) - elif randomChoice == 1: - setBlock(cobblestone, x, 2, z - 1) - setBlock(stone_brick_slab, x, 3, z - 1) - setBlock(stone_brick_slab, x, 2, z) - setBlock(stone_brick_slab, x, 2, z + 1) - elif randomChoice == 2 or randomChoice == 3: - setBlock(red_flower, x, 2, z) - elif j == 34: # Beach - setBlock(sand, x, 1, z) - elif j == 35: # Wetland - randomChoice = randint(0, 2) - if randomChoice == 0: + setBlock(farmland, x, 1, z) + randomChoice = randint(0, 2) + if randomChoice == 0: + setBlock(wheat, x, 2, z) + elif randomChoice == 1: + setBlock(carrots, x, 2, z) + else: + setBlock(potatoes, x, 2, z) + elif j == 32: # Forest setBlock(grass_block, x, 1, z) - else: + randomChoice = randint(0, 8) + if randomChoice >= 0 and randomChoice <= 5: + setBlock(grass, x, 2, z) + elif randomChoice == 6: + fillBlocks(log, x, 2, z, x, 8, z) + fillBlocks(leaves, x - 2, 5, z - 2, x + 2, 6, z + 2) + setBlock(air, x - 2, 6, z - 2) + setBlock(air, x - 2, 6, z + 2) + setBlock(air, x + 2, 6, z - 2) + setBlock(air, x + 2, 6, z + 2) + fillBlocks(leaves, x - 1, 7, z - 1, x + 1, 8, z + 1) + setBlock(air, x - 1, 8, z - 1) + setBlock(air, x - 1, 8, z + 1) + setBlock(air, x + 1, 8, z - 1) + setBlock(air, x + 1, 8, z + 1) + elif j == 33: # Cemetery + setBlock(podzol, x, 1, z) + randomChoice = randint(0, 100) + if randomChoice == 0: + setBlock(cobblestone, x - 1, 2, z) + setBlock(stone_brick_slab, x - 1, 3, z) + setBlock(stone_brick_slab, x, 2, z) + setBlock(stone_brick_slab, x + 1, 2, z) + elif randomChoice == 1: + setBlock(cobblestone, x, 2, z - 1) + setBlock(stone_brick_slab, x, 3, z - 1) + setBlock(stone_brick_slab, x, 2, z) + setBlock(stone_brick_slab, x, 2, z + 1) + elif randomChoice == 2 or randomChoice == 3: + setBlock(red_flower, x, 2, z) + elif j == 34: # Beach + setBlock(sand, x, 1, z) + elif j == 35: # Wetland + randomChoice = randint(0, 2) + if randomChoice == 0: + setBlock(grass_block, x, 1, z) + else: + setBlock(water, x, 1, z) + elif j == 36: # Pitch + setBlock(green_stained_hardened_clay, x, 1, z) + elif j == 37: # Swimming pool + setBlock(water, x, 1, z) + setBlock(white_concrete, x, 0, z) + elif j == 38: # Water setBlock(water, x, 1, z) - elif j == 36: # Pitch - setBlock(green_stained_hardened_clay, x, 1, z) - elif j == 37: # Swimming pool - setBlock(water, x, 1, z) - setBlock(white_concrete, x, 0, z) - elif j == 38: # Water - setBlock(water, x, 1, z) - elif j == 39: # Raw grass - setBlock(grass_block, x, 1, z) - elif j >= 50 and j <= 59: # House corner - building_height = 5 - if j == 51: - building_height = 8 - elif j == 52: - building_height = 11 - elif j == 53: - building_height = 14 - elif j == 54: - building_height = 17 - elif j == 55: - building_height = 20 - elif j == 56: - building_height = 23 - elif j == 57: - building_height = 26 - elif j == 58: - building_height = 29 - elif j == 59: - building_height = 32 + elif j == 39: # Raw grass + setBlock(grass_block, x, 1, z) + elif j >= 50 and j <= 59: # House corner + building_height = 5 + if j == 51: + building_height = 8 + elif j == 52: + building_height = 11 + elif j == 53: + building_height = 14 + elif j == 54: + building_height = 17 + elif j == 55: + building_height = 20 + elif j == 56: + building_height = 23 + elif j == 57: + building_height = 26 + elif j == 58: + building_height = 29 + elif j == 59: + building_height = 32 - fillBlocks(white_concrete, x, 1, z, x, building_height, z) - elif j >= 60 and j <= 69: # House wall - building_height = 4 - if j == 61: - building_height = 7 - elif j == 62: - building_height = 10 - elif j == 63: - building_height = 13 - elif j == 64: - building_height = 16 - elif j == 65: - building_height = 19 - elif j == 66: - building_height = 22 - elif j == 67: - building_height = 25 - elif j == 68: - building_height = 28 - elif j == 69: - building_height = 31 + fillBlocks(white_concrete, x, 1, z, x, building_height, z) + elif j >= 60 and j <= 69: # House wall + building_height = 4 + if j == 61: + building_height = 7 + elif j == 62: + building_height = 10 + elif j == 63: + building_height = 13 + elif j == 64: + building_height = 16 + elif j == 65: + building_height = 19 + elif j == 66: + building_height = 22 + elif j == 67: + building_height = 25 + elif j == 68: + building_height = 28 + elif j == 69: + building_height = 31 - if doorIncrement == 25: - fillBlocks(white_stained_glass, x, 4, z, x, building_height, z) - setBlock(white_concrete, x, 1, z) - setBlock(dark_oak_door_lower, x, 2, z) - setBlock(dark_oak_door_upper, x, 3, z) - doorIncrement = 0 - else: - fillBlocks(white_concrete, x, 1, z, x, 2, z) - fillBlocks(white_stained_glass, x, 3, z, x, building_height, z) - doorIncrement += 1 - setBlock(white_concrete, x, building_height + 1, z) - elif j >= 70 and j <= 79: # House interior - if j >= 70: - setBlock(white_concrete, x, 5, z) - if j >= 71: - setBlock(white_concrete, x, 8, z) - if j >= 72: - setBlock(white_concrete, x, 11, z) - if j >= 73: - setBlock(white_concrete, x, 14, z) - if j >= 74: - setBlock(white_concrete, x, 17, z) - if j >= 75: - setBlock(white_concrete, x, 20, z) - if j >= 76: - setBlock(white_concrete, x, 23, z) - if j >= 77: - setBlock(white_concrete, x, 26, z) - if j >= 78: - setBlock(white_concrete, x, 29, z) + if doorIncrement == 25: + fillBlocks(white_stained_glass, x, 4, z, x, building_height, z) + setBlock(white_concrete, x, 1, z) + setBlock(dark_oak_door_lower, x, 2, z) + setBlock(dark_oak_door_upper, x, 3, z) + doorIncrement = 0 + else: + fillBlocks(white_concrete, x, 1, z, x, 2, z) + fillBlocks(white_stained_glass, x, 3, z, x, building_height, z) + doorIncrement += 1 + setBlock(white_concrete, x, building_height + 1, z) + elif j >= 70 and j <= 79: # House interior + if j >= 70: + setBlock(white_concrete, x, 5, z) + if j >= 71: + setBlock(white_concrete, x, 8, z) + if j >= 72: + setBlock(white_concrete, x, 11, z) + if j >= 73: + setBlock(white_concrete, x, 14, z) + if j >= 74: + setBlock(white_concrete, x, 17, z) + if j >= 75: + setBlock(white_concrete, x, 20, z) + if j >= 76: + setBlock(white_concrete, x, 23, z) + if j >= 77: + setBlock(white_concrete, x, 26, z) if j >= 78: - setBlock(white_concrete, x, 32, z) + setBlock(white_concrete, x, 29, z) + if j >= 78: + setBlock(white_concrete, x, 32, z) - setBlock(glowstone, x, 1, z) + setBlock(glowstone, x, 1, z) - z += 1 - x += 1 - ElementIncr += 1 + z += 1 + x += 1 + ElementIncr += 1 - -print("Saving minecraft world...") -saveRegion() -print( - "Done! Finished in " - + str(round(time.time() - processStartTime, 2)) - + " seconds (" - + str(round((time.time() - processStartTime) / 60, 2)) - + " minutes)" -) -os._exit(0) + print("Saving minecraft world...") + saveRegion() + print( + f"Done! Finished in {(time.time() - processStartTime):.2f} seconds ({((time.time() - processStartTime) / 60):.2f} minutes)" + ) + os._exit(0) diff --git a/src/processData.py b/src/processData.py index af9986a..494f063 100644 --- a/src/processData.py +++ b/src/processData.py @@ -95,21 +95,15 @@ def processData(data, args): del data["elements"][i] if args.debug: - print("Biggest element X: " + str(greatestElementX)) - print("Biggest element Y: " + str(greatestElementY)) - print("Lowest element X: " + str(lowestElementX)) - print("Lowest element Y: " + str(lowestElementY)) + print(f"Biggest element X: {greatestElementX}") + print(f"Biggest element Y: {greatestElementY}") + print(f"Lowest element X: {lowestElementX}") + print(f"Lowest element Y: {lowestElementY}") print( - "Original position determination reference coordinates: " - + str(orig_posDeterminationCoordX) - + ", " - + str(orig_posDeterminationCoordY) + f"Original position determination reference coordinates: {orig_posDeterminationCoordX}, {orig_posDeterminationCoordY}" ) print( - "Map position determination reference coordinates: " - + str(map_posDeterminationCoordX) - + ", " - + str(map_posDeterminationCoordY) + f"Map position determination reference coordinates: {map_posDeterminationCoordX}, {map_posDeterminationCoordY}" ) with open("arnis-debug-processed_data.json", "w", encoding="utf-8") as f: f.write(str(data)) @@ -126,15 +120,7 @@ def processData(data, args): progressPercentage % 10 == 0 and progressPercentage != lastProgressPercentage ): - print( - "Element " - + str(ElementIncr + 1) - + "/" - + str(ElementsLen) - + " (" - + str(progressPercentage) - + "%)" - ) + print(f"Element {ElementIncr + 1}/{ElementsLen} ({progressPercentage}%)") lastProgressPercentage = progressPercentage if element["type"] == "way" and "tags" in element: @@ -605,11 +591,7 @@ def processData(data, args): img[x][y] = imgLanduse[x][y] print( - "Processing finished in " - + str(round(time() - processingStartTime, 2)) - + " seconds (" - + str(round((time() - processingStartTime) / 60, 2)) - + " minutes)" + f"Processing finished in {(time() - processingStartTime):.2f} seconds ({((time() - processingStartTime) / 60):.2f} minutes)" ) if args.debug: imwrite("arnis-debug-map.png", img)