refacotring: rename reduce to cross_file_process and pass the iterator

terryyin · Jun 3, 2018 · ca73858 · ca73858
1 parent 9957dd7
commit ca73858
Show file tree

Hide file tree

Showing 274 changed files with 86,541 additions and 41 deletions.
diff --git a/app.yaml b/app.yaml
@@ -0,0 +1,19 @@
+runtime: python27
+api_version: 1
+threadsafe: true
+
+handlers:
+- url: /favicon\.ico
+  static_files: website/static/favicon.ico
+  upload: website/static/favicon\.ico
+- url: /
+  static_files: index.html
+  upload: index.html
+- url: /website/static
+  static_dir: website/static
+- url: /.*
+  script: website.application
+
+libraries:
+- name: webapp2
+  version: latest
diff --git a/appengine_config.py b/appengine_config.py
@@ -0,0 +1,7 @@
+"""`appengine_config` gets loaded when starting a new application instance."""
+import sys
+import os.path
+# add `lib` subdirectory to `sys.path`, so our `main` module can load
+# third-party libraries.
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib/local'))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib'))
diff --git a/bower.json b/bower.json
@@ -0,0 +1,18 @@
+{
+  "name": "lizard",
+  "description": "lizard",
+  "version": "0.0.0",
+  "homepage": "https://github.com/terryyin/lizard",
+  "license": "MIT",
+  "private": true,
+  "dependencies": {
+    "angular": "1.2.x",
+    "angular-mocks": "~1.2.x",
+    "jquery": "1.10.2",
+    "bootstrap": "~3.1.1",
+    "angular-route": "~1.2.x",
+    "angular-resource": "~1.2.x",
+    "jquery": "1.10.2",
+    "angular-animate": "~1.2.x"
+  }
+}
diff --git a/index.html b/index.html
@@ -0,0 +1,104 @@
+<!DOCTYPE html>
+<html ng-app="lizardApp">
+  <head>
+    <link rel="stylesheet" type="text/css" href="website/static/bower/bootstrap/dist/css/bootstrap.css" />
+    <script src="website/static/bower/angular/angular.js"></script>
+    <script src="website/static/controllers.js"></script>
+  </head>
+  <body style="padding-top: 70px;">
+    <!-- Fixed navbar -->
+    <div class="navbar navbar-inverse" role="navigation">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a href="/" class="nav-brand navbar-left">
+            <img src="website/static/img/logo.png" style="width:40px;" />
+          </a>
+          <a class="navbar-brand" href="#">
+            Lizard <em>code complexity analyzer</em></a>
+
+        </div>
+        <div class="navbar-collapse collapse">
+          <ul class="nav navbar-nav navbar-right">
+            <li><a href="#">Home</a></li>
+            <li><a href="#try">Try Online</a></li>
+            <li><a href="https://github.com/terryyin/lizard">github</a></li>
+            <li><a href="https://pypi.python.org/pypi/lizard">Download</a></li>
+          </ul>
+        </div><!--/.nav-collapse -->
+      </div>
+    </div>
+    <div class="container theme-showcase" role="main">
+      <div class="jumbotron">
+        <h1>Complex is better</h1>
+        <p>Than complicated. It's OK to build very complex software,
+        but you don't have to build it in a complicated way. Lizard is a
+        free open source tool
+        that analyse the complexity of your source code right away supporting many programming languages, without any extra setup.
+        It also does code clone / copy-paste detection.
+        </p>
+        <p><a href="https://pypi.python.org/pypi/lizard" class="btn btn-primary btn-lg" role="button">Download &raquo;</a></p>
+      </div>
+    </div>
+    <div class="row container center-block" ng-controller="SourceFormController as source" ng-submit="submit()">
+      <div class="col-md-6">
+        <div class="panel panel-default">
+          <div class="panel-heading">
+            <h4><a id="try">Try Lizard in Your Browser</a></h4>
+          </div>
+          <div class="panel-body">
+            <form method="post"  role="form" class="form-horizontal">
+              <div class="input-group">
+                <select ng-init="source.lang = source.lang || '.c'"
+                  ng-model="source.lang" class="form-control" ng-options="lang for lang in ['.c','.cpp','.java','.js', '.py', '.m','.h', '.rb', '.swift']"  title="Language">
+                </select>
+                <span class="input-group-btn">
+                  <input type="submit" value="Analyse" class="btn btn-primary">
+                </span>
+              </div>
+              <div><textarea name="content"  class="form-control" rows="10" ng-model="source.code" placeholder="Paste your source code here..."></textarea></div>
+            </form>
+          </div>
+        </div>
+      </div>
+      <div class="col-md-6">
+        <div class="panel panel-default">
+          <div class="panel-heading">
+            <h4>
+              {{progress}}
+            </h4>
+          </div>
+          <div class="panel-body">
+            <span class="label label-info">File Type <span class="badge">{{result.filename}}</span></span>
+            <span class="label label-info">Token Count <span class="badge">{{result.token_count}}</span></span>
+            <span class="label label-info">NLOC <span class="badge">{{result.nloc}}</span></span>
+          </div>
+            <table class="table table-hover">
+              <tr>
+                <th> Function Name </th>
+                <th> NLOC </th>
+                <th> Complexity </th>
+                <th> Token # </th>
+                <th> Parameter # </th>
+              </tr>
+              <tr ng-repeat="fun in result.function_list">
+                <td>{{fun.name}}</td>
+                <td align=right>{{fun.nloc}}</td>
+                <td align=right>{{fun.cyclomatic_complexity}}</td>
+                <td align=right>{{fun.token_count}}</td>
+                <td align=right>{{fun.parameter_count}}</td>
+              </tr>
+            </table>
+        </div>
+      </div>
+      <a href="https://github.com/terryyin/lizard"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://s3.amazonaws.com/github/ribbons/forkme_right_red_aa0000.png" alt="Fork me on GitHub"></a>
+      <script src="website/static/bower/jquery/jquery.min.js"></script>
+      <script src="website/static/bower/bootstrap/dist/js/bootstrap.min.js"></script>
+
+    </body>
+  </html>
diff --git a/index.py b/index.py
@@ -0,0 +1,21 @@
+from flask import Flask, request, url_for, render_template
+from lizard import analyze_file
+import os
+app = Flask(__name__)
+app.config['DEBUG'] = True
+
+
+@app.route('/')
+def hello():
+    return render_template('index.html')
+
+
+@app.route('/analyse', methods=['POST'])
+def analyse():
+    return render_template('index.html', info=analyze_file.analyze_source_code("a.cpp", request.form['content']))
+
+
+@app.errorhandler(404)
+def page_not_found(e):
+    return 'Sorry, nothing at this URL.', 404
+
diff --git a/lizard.py b/lizard.py
@@ -67,11 +67,10 @@ def analyze_files(files, threads=1, exts=None, regression=False):
     result = map_files_to_analyzer(files, file_analyzer, threads)
     if regression:
         result = [r for r in result]
-    for module_info in result:
-        for extension in extensions:
-            if hasattr(extension, 'reduce'):
-                extension.reduce(module_info)
-        yield module_info
+    for extension in extensions:
+        if hasattr(extension, 'cross_file_process'):
+            result = extension.cross_file_process(result)
+    return result
 
 
 def _extension_arg(parser):

diff --git a/lizard_ext/lizardduplicate.py b/lizard_ext/lizardduplicate.py
@@ -211,9 +211,11 @@ def __call__(self, tokens, reader):
             yield token
         reader.context.fileinfo.hash_nodes = list(token_unifier.samples())
 
-    def reduce(self, fileinfo):
-        self.fileinfos.append((len(self.nodes), fileinfo))
-        self.nodes += fileinfo.hash_nodes
+    def cross_file_process(self, fileinfos):
+        for fileinfo in fileinfos:
+            self.fileinfos.append((len(self.nodes), fileinfo))
+            self.nodes += fileinfo.hash_nodes
+            yield fileinfo
 
     def get_duplicates(self, min_duplicate_tokens=70):
         boundaries = [info[0] for info in self.fileinfos]

diff --git a/lizard_ext/lizardduplicated_param_list.py b/lizard_ext/lizardduplicated_param_list.py
@@ -25,16 +25,18 @@ def __init__(self, context=None):
         self.all_count_per_file = Counter()
         super(LizardExtension, self).__init__(context)
 
-    def reduce(self, fileinfo):
-        flt = [
-                f for f in fileinfo.function_list
-                if len(f.parameters) >= DEFAULT_MIN_PARAM_COUNT
-             ]
-        self.all_count.update(self._parameters(f) for f in flt)
-        self.all_count_per_file.update(set(self._parameters(f) for f in flt))
-        for flist in fileinfo.function_list:
-            flist.parameter_list_duplicates = 0
-            flist.parameter_list_duplicated_in_files = 0
+    def cross_file_process(self, fileinfos):
+        for fileinfo in fileinfos:
+            flt = [
+                    f for f in fileinfo.function_list
+                    if len(f.parameters) >= DEFAULT_MIN_PARAM_COUNT
+                 ]
+            self.all_count.update(self._parameters(f) for f in flt)
+            self.all_count_per_file.update(
+                    set(self._parameters(f) for f in flt))
+            for flist in fileinfo.function_list:
+                flist.parameter_list_duplicates = 0
+                flist.parameter_list_duplicated_in_files = 0
 
     def reduce_again(self, fileinfo):
         for flist in fileinfo.function_list:

diff --git a/lizard_ext/lizardio.py b/lizard_ext/lizardio.py
@@ -57,16 +57,18 @@ def _state_global(self, token):
             self.context.current_function.tokens = list()
         self.context.current_function.tokens.append(token)
 
-    def reduce(self, fileinfo):
-        try:
-            new_funcs = {f.unqualified_name: f for f in
-                         fileinfo.function_list}
-            self.all_methods.update(new_funcs)
-            self._add_to_fan_outs(new_funcs.keys())
-            self._add_to_general_fan_out()
-            self._add_to_fan_ins(fileinfo.function_list)
-        except (AttributeError, TypeError, ValueError):
-            pass
+    def cross_file_process(self, fileinfos):
+        for fileinfo in fileinfos:
+            try:
+                new_funcs = {f.unqualified_name: f for f in
+                             fileinfo.function_list}
+                self.all_methods.update(new_funcs)
+                self._add_to_fan_outs(new_funcs.keys())
+                self._add_to_general_fan_out()
+                self._add_to_fan_ins(fileinfo.function_list)
+            except (AttributeError, TypeError, ValueError):
+                pass
+            yield fileinfo
 
     def _add_to_fan_outs(self, keys):
         for other_func in self.all_methods.values():

diff --git a/lizard_ext/lizardwordcount.py b/lizard_ext/lizardwordcount.py
@@ -32,15 +32,17 @@ def __call__(tokens, reader):
                 result[token] = result.get(token, 0) + 1
             yield token
 
-    def reduce(self, fileinfo):
+    def cross_file_process(self, fileinfos):
         '''
         Combine the statistics from each file.
         Because the statistics came from multiple thread tasks. This function
         needs to be called to collect the combined result.
         '''
-        if hasattr(fileinfo, "wordCount"):
-            for k, val in fileinfo.wordCount.items():
-                self.result[k] = self.result.get(k, 0) + val
+        for fileinfo in fileinfos:
+            if hasattr(fileinfo, "wordCount"):
+                for k, val in fileinfo.wordCount.items():
+                    self.result[k] = self.result.get(k, 0) + val
+            yield fileinfo
 
     def print_result(self):
         with open(self.HTML_FILENAME, 'w') as html_file:

diff --git a/templates/index.html b/templates/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+{% autoescape true %}
+<html>
+  <body>
+
+    <form action="{{ url_for('analyse') }}" method="post">
+      <div><textarea name="content" rows="30" cols="60"></textarea></div>
+      <div><input type="submit" value="Analyse"></div>
+    </form>
+
+{% if info %}
+  <ul class=entries>
+  {% for f in info.function_list %}
+    <li><h2>{{ f.name }}</h2>{{ f.nloc }}
+  {% else %}
+    <li><em>Unbelievable.  No entries here so far</em>
+  {% endfor %}
+  </ul>
+ {% endif %}
+  </body>
+</html>
+{% endautoescape %}
diff --git a/test/testExtension.py b/test/testExtension.py
@@ -54,12 +54,11 @@ def setUp(self):
 
     def test_reduce_func_in_lizardio(self):
         lizard_object = None
-        self.assertRaises((AttributeError, TypeError, ValueError), self.ext.reduce(lizard_object))
+        self.assertRaises((AttributeError, TypeError, ValueError), list(self.ext.cross_file_process([lizard_object])))
 
 
     def test_all_extention_simultaneously(self):
-
-        self.ext.reduce(self.lizard_object)
+        list(self.ext.cross_file_process([self.lizard_object]))
         correct_max_nd = {'foo': 1,
                           'bar': 0}
         correct_max_nested_struct = {'foo': 1,

diff --git a/test/testWordCountPlugin.py b/test/testWordCountPlugin.py
@@ -51,8 +51,7 @@ def test_should_not_line_continuer(self):
 
     def test_reduce_the_result(self):
         list(self.ext(["a"], self.reader))
-        self.ext.reduce(self.reader.fileinfo)
-        self.ext.reduce(self.reader.fileinfo)
+        list(self.ext.cross_file_process([self.reader.fileinfo, self.reader.fileinfo]))
         self.assertEqual(2, self.ext.result['a'])
 
 class TestWordCountOutput(unittest.TestCase):

diff --git a/test/test_extensions/testDuplicate.py b/test/test_extensions/testDuplicate.py
@@ -12,9 +12,9 @@ def setUp(self):
         self.builder = CFunctionBuilder()
 
     def detect(self, code, min_duplicate_tokens = 31):
-        self.detector.reduce(
-                get_cpp_fileinfo_with_extension(code, self.detector)
-            )
+        list(self.detector.cross_file_process(
+                [get_cpp_fileinfo_with_extension(code, self.detector)]
+            ))
         return list(self.detector.get_duplicates(min_duplicate_tokens))
 
     def test_empty_file(self):

diff --git a/test/test_extensions/testFanInFanOut.py b/test/test_extensions/testFanInFanOut.py
@@ -11,7 +11,7 @@ def fanio(*code):
     ext = FanInOut()
     def process(ext, code):
         result = FileAnalyzer(get_extensions([ext])).analyze_source_code("a.cpp", code)
-        ext.reduce(result)
+        list(ext.cross_file_process([result]))
         return result
     results = [process(ext, code) for code in code]
     return results[0].function_list[0]

diff --git a/website/__init__.py b/website/__init__.py
@@ -0,0 +1,19 @@
+import webapp2
+import json
+from json import JSONEncoder
+class FileInfoEncoder(JSONEncoder):
+        def default(self, o):
+                    return o.__dict__
+from lizard import analyze_file
+
+class LizardAnalyzer(webapp2.RequestHandler):
+    def post(self):
+        dictionary = json.loads(self.request.body)
+        file_info = analyze_file.analyze_source_code(dictionary.get("lang", ".c"), dictionary.get("code", ""))
+        self.response.headers['Content-Type'] = 'application/json'
+        self.response.write(json.dumps(file_info, cls=FileInfoEncoder))
+
+application = webapp2.WSGIApplication([
+    ('/analyze', LizardAnalyzer),
+], debug=True)
+
diff --git a/website/static/bower/angular-animate/.bower.json b/website/static/bower/angular-animate/.bower.json
@@ -0,0 +1,18 @@
+{
+  "name": "angular-animate",
+  "version": "1.2.18",
+  "main": "./angular-animate.js",
+  "dependencies": {
+    "angular": "1.2.18"
+  },
+  "homepage": "https://github.com/angular/bower-angular-animate",
+  "_release": "1.2.18",
+  "_resolution": {
+    "type": "version",
+    "tag": "v1.2.18",
+    "commit": "cd5a97b990643e831802256cbfd3c39f0372fd3b"
+  },
+  "_source": "git://github.com/angular/bower-angular-animate.git",
+  "_target": "~1.2.x",
+  "_originalSource": "angular-animate"
+}