Merge pull request #128 from arnav-mandal1234/gsoc_similarity_matching

Similarity Matching by Fingerprint Comparison.

src/deltacode/__init__.py

@@ -39,6 +39,7 @@
     # package is not installed ??
     __version__ = '1.0.0'
 
+SIMILARITY_LIMIT = 35
 
 class DeltaCode(object):
     """
@@ -60,6 +61,7 @@ def __init__(self, new_path, old_path, options):
             self.license_diff()
             self.copyright_diff()
             self.stats.calculate_stats()
+            self.similarity()
             # Sort deltas by score, descending, i.e., high > low, and then by
             # factors, alphabetically.  Run the least significant sort first.
             self.deltas.sort(key=lambda Delta: Delta.factors, reverse=False)
@@ -80,6 +82,28 @@ def align_scans(self):
             for f in self.old.files:
                 f.original_path = f.path
 
+    def similarity(self):
+        """
+        Compare the fingerprints of a pair of 'new' and 'old' File objects
+        in a Delta object and change the Delta object's 'score' attribute --
+        and add an appropriate category 'Similar with hamming distance'
+        to the Delta object's 'factors' attribute -- if the hamming
+        distance is less than the threshold distance.
+        """
+        for delta in self.deltas:
+            if delta.new_file == None or delta.old_file == None:
+                continue
+            new_fingerprint = delta.new_file.fingerprint
+            old_fingerprint = delta.old_file.fingerprint
+            if new_fingerprint == None or old_fingerprint == None:
+                continue
+            new_fingerprint = utils.bitarray_from_hex(delta.new_file.fingerprint)
+            old_fingerprint = utils.bitarray_from_hex(delta.old_file.fingerprint)
+            hamming_distance = utils.hamming_distance(new_fingerprint, old_fingerprint)
+            if hamming_distance > 0 and hamming_distance <= SIMILARITY_LIMIT:
+                delta.score += hamming_distance
+                delta.factors.append('Similar with hamming distance : {}'.format(hamming_distance))
+
     def determine_delta(self):
         """
         Add to a list of Delta objects that can be sorted by their attributes,

src/deltacode/models.py

@@ -179,6 +179,7 @@ def __init__(self, dictionary={}):
         self.name = dictionary.get('name', '')
         self.size = dictionary.get('size', '')
         self.sha1 = dictionary.get('sha1', '')
+        self.fingerprint = dictionary.get('fingerprint', '')
         self.original_path = ''
         self.licenses = self.get_licenses(dictionary) if dictionary.get('licenses') else []
         self.copyrights = self.get_copyrights(dictionary) if dictionary.get('copyrights') else []
@@ -210,6 +211,7 @@ def to_dict(self):
             ('name', self.name),
             ('size', self.size),
             ('sha1', self.sha1),
+            ('fingerprint', self.fingerprint),
             ('original_path', self.original_path),
         ])
 

src/deltacode/test_utils.py

@@ -0,0 +1,170 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/deltacode/
+# The DeltaCode software is licensed under the Apache License version 2.0.
+# Data generated with DeltaCode require an acknowledgment.
+# DeltaCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with DeltaCode or any DeltaCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with DeltaCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  DeltaCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  DeltaCode is a free and open source software analysis tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/deltacode/ for support and download.
+#
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import division
+from __future__ import unicode_literals
+
+from collections import OrderedDict
+
+import io
+import json
+
+from commoncode.system import on_windows
+
+
+def run_scan_click(options, monkeypatch=None, test_mode=True, expected_rc=0, env=None):
+    """
+    Run a scan as a Click-controlled subprocess
+    If monkeypatch is provided, a tty with a size (80, 43) is mocked.
+    Return a click.testing.Result object.
+    """
+    import click
+    from click.testing import CliRunner
+    from deltacode import cli
+
+    options = add_windows_extra_timeout(options)
+
+    if monkeypatch:
+        monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True)
+        monkeypatch.setattr(click , 'get_terminal_size', lambda : (80, 43,))
+    runner = CliRunner()
+
+    result = runner.invoke(cli.cli, options, catch_exceptions=False, env=env)
+
+    output = result.output
+    if result.exit_code != expected_rc:
+        opts = get_opts(options)
+        error = '''
+Failure to run: deltacode %(opts)s
+output:
+%(output)s
+''' % locals()
+        assert result.exit_code == expected_rc, error
+    return result
+
+
+def get_opts(options):
+    try:
+        return ' '.join(options)
+    except:
+        try:
+            return b' '.join(options)
+        except:
+            return b' '.join(map(repr, options))
+
+
+WINDOWS_CI_TIMEOUT = '222.2'
+
+
+def add_windows_extra_timeout(options, timeout=WINDOWS_CI_TIMEOUT):
+    """
+    Add a timeout to an options list if on Windows.
+    """
+    if on_windows and '--timeout' not in options:
+        # somehow the Appevyor windows CI is now much slower and timeouts at 120 secs
+        options += ['--timeout', timeout]
+    return options
+
+
+def check_json_scan(expected_file, result_file, regen=False, remove_file_date=False, ignore_headers=False):
+    """
+    Check the scan `result_file` JSON results against the `expected_file`
+    expected JSON results.
+
+    If `regen` is True the expected_file WILL BE overwritten with the new scan
+    results from `results_file`. This is convenient for updating tests
+    expectations. But use with caution.
+
+    if `remove_file_date` is True, the file.date attribute is removed.
+    """
+    results = load_json_result(result_file, remove_file_date)
+    if regen:
+        with open(expected_file, 'wb') as reg:
+            json.dump(results, reg, indent=2, separators=(',', ': '))
+
+    expected = load_json_result(expected_file, remove_file_date)
+
+    if ignore_headers:
+        results.pop('headers', None)
+        expected.pop('headers', None)
+
+    # NOTE we redump the JSON as a string for a more efficient display of the
+    # failures comparison/diff
+    # TODO: remove sort, this should no longer be needed
+    expected = json.dumps(expected, indent=2, sort_keys=True, separators=(',', ': '))
+    results = json.dumps(results, indent=2, sort_keys=True, separators=(',', ': '))
+    assert expected == results
+
+
+def load_json_result(location, remove_file_date=False):
+    """
+    Load the JSON scan results file at `location` location as UTF-8 JSON.
+
+    To help with test resilience against small changes some attributes are
+    removed or streamlined such as the  "tool_version" and scan "errors".
+
+    To optionally also remove date attributes from "files" and "headers"
+    entries, set the `remove_file_date` argument to True.
+    """
+    with io.open(location, encoding='utf-8') as res:
+        scan_results = res.read()
+    return load_json_result_from_string(scan_results, remove_file_date)
+
+
+def load_json_result_from_string(string, remove_file_date=False):
+    """
+    Load the JSON scan results `string` as UTF-8 JSON.
+    """
+    scan_results = json.loads(string, object_pairs_hook=OrderedDict)
+    # clean new headers attributes
+    streamline_headers(scan_results)
+
+    scan_results['deltas'].sort(key=lambda  x: x['factors'], reverse=False)
+    scan_results['deltas'].sort(key=lambda x: x['score'], reverse=True)
+    return scan_results
+
+
+def streamline_errors(errors):
+    """
+    Modify the `errors` list in place to make it easier to test
+    """
+    for i, error in enumerate(errors[:]):
+        error_lines = error.splitlines(True)
+        if len(error_lines) <= 1:
+            continue
+        # keep only first and last line
+        cleaned_error = ''.join([error_lines[0] + error_lines[-1]])
+        errors[i] = cleaned_error
+
+
+def streamline_headers(headers):
+    """
+    Modify the `headers` list of mappings in place to make it easier to test.
+    """
+    headers.pop('deltacode_version', None)
+    headers.pop('deltacode_options', None)
+    streamline_errors(headers['deltacode_errors'])

src/deltacode/utils.py

@@ -25,14 +25,15 @@
 
 from __future__ import absolute_import, division
 
+from bitarray import bitarray
 from collections import defaultdict
+from bitarray import bitdiff
 
+import binascii
 import os
 
-
 from commoncode import paths
 
-
 def update_from_license_info(delta, unique_categories):
     """
     Increase an 'added' or 'modified' Delta object's 'score' attribute and add
@@ -282,3 +283,33 @@ def get_notice():
     notice = acknowledgment_text.strip().replace('  ', '')
 
     return notice
+
+
+def hamming_distance(fingerprint1, fingerprint2):
+    """
+    Return hamming distance between two given fingerprints.
+    Hamming distance is the difference in the bits of two binary string.
+    Files with fingerprints whose hamming distance are less tends to be more similar.
+    """
+    distance = bitdiff(fingerprint1, fingerprint2)
+    result = int(distance)
+
+    return result
+
+def bitarray_from_hex(fingerprint_hex):
+    """
+    Return bitarray from a hex string.
+    """
+    bytes = binascii.unhexlify(fingerprint_hex)
+    result = bitarray_from_bytes(bytes)
+
+    return result
+
+def bitarray_from_bytes(b):
+    """
+    Return bitarray from a byte string, interpreted as machine values.
+    """
+    a = bitarray()
+    a.frombytes(b)
+
+    return a

tests/data/cli/scan_1_file_moved_new.json

@@ -71,6 +71,7 @@
       "size": 200,
       "sha1": "84b647771481d39dd3a53f6dc210c26abac37748",
       "md5": "3cb56efa7140478458dbaa2b30239845",
+      "fingerprint": "83ee5b9e652faad754b8e20bead792f8",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -130,6 +131,7 @@
       "size": 200,
       "sha1": "310797523e47db8481aeb06f1634317285115091",
       "md5": "19efdad483f68bc9997a5c1f7ba41b26",
+      "fingerprint": "e30cf09443e7878dfcd3288886e97533",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -189,6 +191,7 @@
       "size": 200,
       "sha1": "fd5d3589c825f448546d7dcec36da3e567d35fe9",
       "md5": "795ff2cae8ece792a9bfebe18ad3c8e6",
+      "fingerprint": "e30cf09443e7878dfed3288886e97533",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -248,6 +251,7 @@
       "size": 200,
       "sha1": "6f71666c46446c29d3f45feef5419ae76fb86a5b",
       "md5": "fc403815d6605df989414ff40a64ea17",
+      "fingerprint": "e30cf09443e7878dfed3288886e97542",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -307,6 +311,7 @@
       "size": 200,
       "sha1": "70f6ce80985578b5104db0abc578cf5a05e78f4b",
       "md5": "af9e9420c6c5b2b0ca74e96bf7c1a2f4",
+      "fingerprint": "e30cf09443e7878dfed1088886e97542",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -366,6 +371,7 @@
       "size": 200,
       "sha1": "3340d86b1da9323067db8022f86dc97cfccee1d0",
       "md5": "74ce2b26cebb32670634270dde1fdf33",
+      "fingerprint": "e30cf09443e7878dfed1088886e97232",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -425,6 +431,7 @@
       "size": 200,
       "sha1": "e49d4463662414bee5ad2d2e5c1fbd704f33b84e",
       "md5": "8d458f54d32959b03dff37ef485b29c6",
+      "fingerprint": "e30cf83443e7878dfed1088886e97232",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",
@@ -484,6 +491,7 @@
       "size": 200,
       "sha1": "98c9e6bed78b1513c28e666016cb35a50708c36e",
       "md5": "c3559aef44883a46e007ab01213091cb",
+      "fingerprint": "e30cf83443e7878dfed1032386e97232",
       "files_count": null,
       "mime_type": "text/plain",
       "file_type": "ASCII text, with CRLF line terminators",