diff --git a/Makefile.in b/Makefile.in index e745bc4f..0f2e7877 100644 --- a/Makefile.in +++ b/Makefile.in @@ -850,7 +850,7 @@ install-data-hook: cp internal.net $(DESTDIR)/$(prefix)/etc/joy mkdir -p $(DESTDIR)/$(prefix)/include/joy # SLEUTHVER= grep version setup.py | awk -F\' '{print $2}' - (cd sleuth_pkg && python setup.py bdist --format=gztar) + (cd sleuth_pkg && python setup.py bdist --format=gztar && pip install .) tar -xf $(SLEUTHFILE) -C $(DESTDIR) # Tell versions [3.59,3.63) of GNU make to not export all variables. diff --git a/sleuth_pkg/sleuth/__init__.py b/sleuth_pkg/sleuth/__init__.py index 5ac369cc..90c14404 100644 --- a/sleuth_pkg/sleuth/__init__.py +++ b/sleuth_pkg/sleuth/__init__.py @@ -1,6 +1,6 @@ from sleuth import * -from api_joy import * -from enrich_tls import enrich_tls -from fingerprint import * -from inferences import * +from sleuth.api_joy import * +from sleuth.enrich_tls import enrich_tls +from sleuth.fingerprint import * +from sleuth.inferences import * diff --git a/sleuth_pkg/sleuth/api_joy.py b/sleuth_pkg/sleuth/api_joy.py index d0a79bdd..a235c449 100644 --- a/sleuth_pkg/sleuth/api_joy.py +++ b/sleuth_pkg/sleuth/api_joy.py @@ -43,6 +43,9 @@ from .sleuth import DictStreamIteratorFromFile from .sleuth import DictStreamIterator from .sleuth import SleuthFileType +from .sleuth import DictStreamProcessor +from .sleuth import DictStreamFilterIterator +from .sleuth import SleuthPredicate class FlowIteratorFromFile(DictStreamIteratorFromFile): @@ -126,8 +129,8 @@ def __init__(self, source): self.flows = iter(self.active_flows.values()) - def next(self): - return self.flows.next() + def __next__(self): + return self.flows.__next__() # merge f2 into f1, where both flows are in the same direction, and # f1 precedes f2 (f1.ts < f2.ts) @@ -187,8 +190,8 @@ def __init__(self, source): def __iter__(self): return self - def next(self): - flow = self.source.next() + def __next__(self): + flow = self.source.__next__() # # if flow is not dns, check cache for response returning destination address # diff --git a/sleuth_pkg/sleuth/fingerprint.py b/sleuth_pkg/sleuth/fingerprint.py index c99966ca..bb0803f8 100644 --- a/sleuth_pkg/sleuth/fingerprint.py +++ b/sleuth_pkg/sleuth/fingerprint.py @@ -35,7 +35,7 @@ """ import os import json -from sleuth import SleuthTemplateDict +from .sleuth import SleuthTemplateDict class fingerprinter(object): fingerprint_dict = { diff --git a/sleuth_pkg/sleuth/inferences.py b/sleuth_pkg/sleuth/inferences.py index 60446d78..53727aef 100644 --- a/sleuth_pkg/sleuth/inferences.py +++ b/sleuth_pkg/sleuth/inferences.py @@ -52,11 +52,11 @@ def tls_fp_dict_init(): with open(tls_fp_path) as f: for counter, line in enumerate(f): tmp = json.loads(line) - #print json.dumps(tmp) + #print(json.dumps(tmp)) fpvalue = json.dumps(tmp['str_repr']) fpvalue = fpvalue.strip('"') if fpvalue in tls_fp_dict: - print "warning: duplicate tls fingerprint in line " + str(counter + 1) + " of file " + tls_fp_file + print("warning: duplicate tls fingerprint in line " + str(counter + 1) + " of file " + tls_fp_file) tls_fp_dict[fpvalue] = tmp @@ -121,7 +121,7 @@ def hex_fp_normalize(s): def element_is_parent(s): if s: - if s[0] is '(' and s[1] is '(': + if s[0] == '(' and s[1] == '(': return True else: return False @@ -129,22 +129,22 @@ def element_is_parent(s): return False def get_next_element(s): - if s is '': + if s == '': return '', '', 0 - if s[0] is ')': + if s[0] == ')': level = 0 for c in s: - if c is not ')': + if c != ')': break; level = level + 1 return '', '', -level if True: level = 0 - while s[level] is '(': + while s[level] == '(': level = level + 1 - if level is 0: + if level == 0: return '', '', 0 tmp = string.split(s[level:], ')', 1) @@ -153,10 +153,10 @@ def get_next_element(s): def print_out_structured_data(s): current_level = 0 - while s is not '': + while s != '': element, s, level = get_next_element(s) current_level += level - print current_level, element, s + print(current_level, element, s) def structured_fp_normalize(s): @@ -179,7 +179,7 @@ def structured_fp_normalize(s): # parse client extensions, if present output += '(' - while s is not '' and s is not ')': + while s != '' and s != ')': element, s, level = get_next_element(s) typecode = element[0:4] data = element[4:] @@ -194,7 +194,7 @@ def tls_inference(f, kwargs): if not tls_fp_dict: tls_fp_dict_init() - # print json.dumps(tls_fp_dict) + # print(json.dumps(tls_fp_dict)) if 'fingerprints' in f: if 'tls' in f['fingerprints']: diff --git a/sleuth_pkg/sleuth/sleuth.py b/sleuth_pkg/sleuth/sleuth.py index b9c16fa5..4a3643b4 100644 --- a/sleuth_pkg/sleuth/sleuth.py +++ b/sleuth_pkg/sleuth/sleuth.py @@ -55,7 +55,7 @@ def __init__(self): def __iter__(self): return self - def next(self): + def __next__(self): n = dict() return n @@ -97,7 +97,7 @@ def _load_file(self): else: self.f = open(self.file_name, 'r') - def next(self): + def __next__(self): while True: try: line = self.f.readline() @@ -129,15 +129,15 @@ def __init__(self, source, filter): self.source = source self.filter = filter - def next(self): + def __next__(self): """ Find the next JSON object from source that matches the given filter :return: """ - tmp = self.source.next() + tmp = self.source.__next__() while self.filter.match(tmp) is not True: - tmp = self.source.next() + tmp = self.source.__next__() return tmp @@ -147,9 +147,9 @@ def __init__(self, source, elements): self.source = source self.template = SleuthTemplateDict(elements) - def next(self): + def __next__(self): while True: - tmp = self.source.next() + tmp = self.source.__next__() output = self.template.copy_selected_elements(self.template.template, tmp) if output: return output @@ -160,8 +160,8 @@ def __init__(self, source, elements): self.source = source self.template = SleuthTemplateDict(elements) - def next(self): - tmp = self.source.next() + def __next__(self): + tmp = self.source.__next__() output = self.template.normalize_selected_elements(self.template.template, tmp) return output @@ -172,8 +172,8 @@ def __init__(self, source, elements, func): self.template = SleuthTemplateDict(elements) self.func = func - def next(self): - tmp = self.source.next() + def __next__(self): + tmp = self.source.__next__() output = self.template.apply_to_selected_elements(self.template.template, tmp, self.func) return output @@ -186,8 +186,8 @@ def __init__(self, source, name, function, **kwargs): self.function = function self.kwargs = kwargs - def next(self): - nextval = self.source.next() + def __next__(self): + nextval = self.source.__next__() tmp = self.function(nextval, self.kwargs) if tmp: nextval[self.name] = tmp @@ -201,8 +201,8 @@ def __init__(self, source, name, function, **kwargs): self.function = function self.kwargs = kwargs - def next(self): - nextval = self.source.next() + def __next__(self): + nextval = self.source.__next__() tmp = self.function(nextval, self.kwargs) if tmp: if self.name not in nextval: @@ -221,9 +221,9 @@ def __init__(self, source, indent=None): def __iter__(self): return self - def next(self): + def __next__(self): try: - obj = self.source.next() + obj = self.source.__next__() value = pickle.dumps(obj) self.key = tuple(obj.keys()) if value in self.dist: @@ -244,7 +244,7 @@ def next(self): for d in output: json.dump(d, sys.stdout, indent=self.indent) - print "" + print("") raise StopIteration @@ -277,7 +277,7 @@ def post_process(self, proc=None): for obj in self.obj_set: try: json.dump(obj, sys.stdout, indent=self.indent) - print "" + print("") except IOError: # Broken pipe, exit loop break @@ -301,7 +301,7 @@ def main_process(self, obj): def post_process(self, proc=None): if self.context: - print self.context + print(self.context) for k, v in self.dict.items(): v.post_process(copy.deepcopy(proc)) @@ -348,7 +348,7 @@ def post_process(self): # NOTE: sum_over might interfere with --dist d["sum_over"] = self.total json.dump(d, sys.stdout, indent=self.indent) - print "" + print("") class DictStreamDistributionProcessor(DictStreamProcessor): @@ -379,7 +379,7 @@ def post_process(self): for d in output: json.dump(d, sys.stdout, indent=self.indent) - print "" + print("") """ @@ -413,7 +413,7 @@ def string_to_template_object(self, s): if needArg: t += "None" t += '}' - #print "t: " + t + #print("t: " + t) return eval(t) def copy_selected_elements(self, tmplDict, obj): @@ -543,9 +543,9 @@ def __init__(self, elements): self.matchAll = True def eval(self, flow): - # print 'flow: ' + str(flow) - # print 'op: ' + str(self.op) - # print 'arg: ' + str(self.arg) + # print('flow: ' + str(flow)) + # print('op: ' + str(self.op)) + # print('arg: ' + str(self.arg)) # If flow is list, match any element in it if isinstance(flow, list): @@ -561,8 +561,9 @@ def eval(self, flow): listMatch = True return listMatch elif isinstance(flow, dict): - # print 'dict flow: ' + str(flow) - x = flow.values()[0] + # print('dict flow: ' + str(flow)) + + x = list(flow.values())[0] return self.eval(x) if self.op == '=': @@ -571,9 +572,9 @@ def eval(self, flow): elif isinstance(self.arg, int): return self.arg == flow else: - # print '------------------' - # print 'flow: ' + str(flow) - # print 'arg: ' + str(self.arg) + # print('------------------') + # print('flow: ' + str(flow)) + # print('arg: ' + str(self.arg) return fnmatch.fnmatch(flow, self.arg) elif self.op == '~': if self.arg == '*': @@ -593,7 +594,7 @@ def match(self, flow): else: output = self.template.get_selected_element(self.template.template, flow) if output: - return self.eval(output.values()[0]) + return self.eval(list(output.values())[0]) else: if self.op == '~' and self.arg == '*': # True because element is absent from flow