Merge pull request #108 from Str3am786/soca_3.10

0.0.2 Closing Issues

src/soca/__main__.py

@@ -63,10 +63,11 @@ def fetch(input, output, repo_type, not_archived, not_forked, not_disabled):
 @click.option('--output','-o', default="repos-metadata", help="Dir where repositories metadata will be saved", metavar='<path>')
 @click.option('--inspect4py','-i4p', is_flag=True, help="Use inspect4py to extract additional metadata from Python repositories")
 @click.option('--verbose','-v', flag_value=True, default=False, show_default=True, help="Fetch only repos that are not archived")
-def extract(input, output, inspect4py, verbose):
+@click.option('--keep','-k', flag_value=True, default=False, show_default=True, help="Keep or not previous output directory")
+def extract(input, output, inspect4py, verbose, keep):
     """Fetch and save metadata from introduced repos"""
     from soca.commands import extract_metadata
-    extract_metadata.extract(input, output, inspect4py, verbose)
+    extract_metadata.extract(input, output, inspect4py, verbose, keep)
 
 @cli.command()
 @click.option('--input','-i', required=True, help="Dir repositories metadata in json format", metavar='<dir-json-metadata>')

src/soca/assets/about.html

@@ -66,6 +66,7 @@ <h3 style="margin-bottom: 1rem;">Team</h3>
 				<li><strong>Daniel Garijo</strong>: mentor. Distinguished researcher at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
 				<li><strong>Esteban Gonz&aacute;lez</strong>: mentor. PhD student at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
 				<li><strong>Oscar Corcho</strong>: mentor. Full professor at <i>Universidad Polit&eacute;cnica de Madrid, ETSIINF</i></li>
+			</ul>
 		</p>
     </article>
 	<article class="soca-card" style="min-height: 26rem; display: inline; min-width: 56.5rem;">

src/soca/assets/soca-card.css

@@ -19,7 +19,7 @@
 .soca-card .tooltip.bottom-right .tooltip-arrow{top:0;left:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#3e3e3e}
 .soca-card .tooltip-inner{max-width:200px;padding:3px 8px;color:#e0e0e0;text-align:center;background-color:#3e3e3e;border-radius:4px}
 .soca-card .tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}
-.soca-card {display: flex; flex-direction: column; justify-content: space-between; line-height: 1.42857143; margin: 0.2rem; min-width: 28rem; max-width: 28rem; min-height: 15rem; max-height: 15rem; padding: 1.3rem; border-radius: 19px; background-color: #e0e0e0; box-shadow: inset 5px 5px 10px #bebebe, inset -5px -5px 10px #ffffff;}
+.soca-card {display: flex; flex-direction: column; justify-content: space-between; line-height: 1.42857143; margin: 0.2rem; min-width: 28rem; max-width: 28rem; min-height: 15rem; max-height: 20rem; padding: 1.3rem; border-radius: 19px; background-color: #e0e0e0; box-shadow: inset 5px 5px 10px #bebebe, inset -5px -5px 10px #ffffff;}
 .soca-card .repo-logo{width: 5.5rem;}
 .soca-card .description{text-align: justify; overflow: hidden; text-overflow: ellipsis; display: -webkit-box; -webkit-line-clamp: 6; line-clamp: 6; -webkit-box-orient: vertical;}
 .soca-card .recently-updated{width: 1.4rem; height: 1.4rem; border-radius: 50%; margin-left: auto; margin-right: 0.1rem; background-color: #3e3e3e;}

src/soca/commands/extract_metadata.py

@@ -14,19 +14,24 @@
 import pprint
 
 
-def extract(repos_csv, output, use_inspect4py, verbose):
+def extract(repos_csv, output, use_inspect4py, verbose, keep):
     """
     @Param repos_csv: input file from the fetch command, A list of github urls
     @Param output: defined output file
     @Param use_inspect4py: Bool to indicate desire to use inspect4py
     @Param verbose: Bool to choose whether or not to Fetch only repos that are not archived
+    @Param keep: Bool to choose whether or not to wipe the output directory. If false it will delete
 
     Returns:
     @return: folder with a json file per repo url within repos.csv
     """
-    # Make output dir
-    if not os.path.exists(output):
+    # Deletes repo if it exists. Ensures no merging of abandoned json files.
+    if os.path.exists(output) and not keep:
+        shutil.rmtree(output)
         os.makedirs(output)
+    elif not os.path.exists(output):
+        os.makedirs(output)
+
 
     if os.path.isfile(repos_csv):
         with open(repos_csv) as repos:

src/soca/commands/portal/card.py

@@ -9,6 +9,7 @@
 def cards_data_dump(repo_metadata_dir):
 
     cards_data = []
+    failed_cards = []
 #TODO change
     #add possible "final release"
     for file in os.listdir(os.fsencode(repo_metadata_dir)):
@@ -18,9 +19,9 @@ def cards_data_dump(repo_metadata_dir):
                 print(f"Creating card for '{filename}'")
                 repo_metadata = json.load(json_metadata)
                 md = metadata.metadata(repo_metadata_dir, repo_metadata)
-                citations = md.citations()
-                print(md.identifier())
-                cards_data.append({
+                try:
+                    citations = md.citations()
+                    cards_data.append({
                     'id': md.repo_url(),
                     'html_card': html_view(repo_metadata_dir, repo_metadata, False),
                     'html_card_embedded': html_view(repo_metadata_dir, repo_metadata, True),
@@ -56,10 +57,13 @@ def cards_data_dump(repo_metadata_dir):
                     'isWeb': md.repo_type() == 'web',
                     'owner': md.owner()
                 })
-
+                except:
+                    print("ERROR")
+                    print(filename)
+                    print("=======")
+                    failed_cards.append(filename)
     print('-'*80)
-
-    return cards_data
+    return [cards_data,failed_cards]
 
 
 def html_view(repo_metadata_dir, repo_metadata, embedded, minify=True):

src/soca/commands/portal/metadata.py

@@ -18,7 +18,7 @@ class metadata(object):
     def __init__(self, repo_metadata_dir, repo_metadata, embedded = False):
         self.repo_metadata_dir = os.path.abspath(repo_metadata_dir)
         self.md = repo_metadata
-        self.base = 'https://github.com/dakixr/soca/raw/main/src/soca/assets/' if embedded else ''
+        self.base = 'https://github.com/oeg-upm/soca/tree/main/src/soca/assets' if embedded else ''
 
     # Assets ####################################################
     def logo(self):
@@ -684,6 +684,7 @@ def of_correctType(result, ofType):
     else:
         raise Exception("not of correct %s type" %ofType)
 
+
 class citation_parser(object):
 
     def __init__(self, citation) -> None:

src/soca/commands/portal/portal.py

@@ -21,12 +21,26 @@ def generate(repo_metadata_dir, output, title, favicon):
     with open(f"{base_dir}/assets/template.html") as template:
         soup = BeautifulSoup(template.read(), features="html.parser")
 
+    cards_and_failed = card.cards_data_dump(repo_metadata_dir)
+    cards_data = cards_and_failed[0]
+    failed_cards = cards_and_failed[1]
     # Create json with cards data and metadata
-    cards_data = card.cards_data_dump(repo_metadata_dir)
     with open(f"{output}/cards_data.json", "w") as cards_data_json:
         json.dump(cards_data ,fp=cards_data_json, indent=4)
         print(f"✅ A total of {len(cards_data)} cards saved in: {os.path.abspath(output)}/cards_data.json")
-
+        #if there are failed cards:
+        if (num_failed:=len(failed_cards))>0:
+            print('\033[91m' + f"There has been {num_failed} repositories that have failed while creating a card. Check log file" + '\033[0m')
+            try:
+                log_file = "failed_repos.log"
+                with open(f"{output}/" + log_file, "w") as log:
+                    for repo_name in failed_cards:
+                        log.write(repo_name + "\n")
+                        print(f"✅ Log file saved in: {os.path.abspath(output)}/failed_repos.log")
+            except:
+                Exception("Failed to write Log file")
+        else:
+            print("With no failures!")
     # Insert extra scripts
     sc = scripts.scripts()
     soup.body.script.string = sc.function_copy_card() + sc.function_tooltip() + sc.functions_modals()