fixed feeder logging and added upd/del/add functionality for feed config

.gitignore

@@ -17,6 +17,7 @@ db_repository/
 .DS_Store
 redis-stable/
 *.log
+feeder/feed.json
 
 
 celerybeat-schedule

app/__init__.py

@@ -18,14 +18,9 @@
     handler = RotatingFileHandler('tmp/ostip_access.log', 'a', 1 * 1024 * 1024, 10)
     a_logger.addHandler(handler)
 
-    #feeder logs
-    feed_logger = logging.getLogger('feeder')
-    handler = RotatingFileHandler('tmp/ostip_feeds.log', 'a', 1 * 1024 * 1024, 10)
-    feed_logger.addHandler(handler)
-
     #error/app info logs
     file_handler = RotatingFileHandler('tmp/ostip.log', 'a', 1 * 1024 * 1024, 10)
-    file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
+    file_handler.setFormatter(logging.Formatter('%(asctime)s %(module)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
     app.logger.setLevel(logging.INFO)
     file_handler.setLevel(logging.INFO)
     app.logger.addHandler(file_handler)

app/forms.py

@@ -3,7 +3,22 @@
 from wtforms.validators import DataRequired
 from wtforms.widgets import TextArea, HiddenInput
 from wtforms.ext.sqlalchemy.fields import QuerySelectField
-from .models import Source, Tlp, Level, Itype, Control, Status, Likelihood
+from .models import Source, Tlp, Level, Itype, Control, Status, Likelihood, Event
+
+
+class FeedConfigForm(Form):
+    index_id = IntegerField(widget=HiddenInput())
+    name = StringField('Name', validators=[DataRequired()])
+    frequency = StringField('Frequency', validators=[DataRequired()])
+    event = QuerySelectField('Event',
+                                  query_factory=lambda: Event.query.join(Source).filter(Source.name == 'Feed'),
+                                  get_label='name')
+    module = details = StringField('Modules', widget=TextArea(), validators=[DataRequired()])
+
+
+
+
+
 
 
 class EventForm(Form):

app/templates/base.html

@@ -47,8 +47,8 @@
                         <li {% if request.path == '/admin/data_types/view' %} class="active" {% endif %}>
                             <a href="/admin/data_types/view">Data Types</a>
                         </li>
-                        <li {% if request.path == '/feeds/config' %} class="active" {% endif %}>
-                            <a href="/feeds/config">Feeds</a>
+                        <li {% if request.path == '/feeds/config/view' %} class="active" {% endif %}>
+                            <a href="/feeds/config/view">Feeds</a>
                         </li>
                     </ul>
                 </div>

app/templates/event_view.html

@@ -125,10 +125,11 @@ <h3 class="panel-title">{{ note.created.strftime('%Y-%m-%d %H:%M') }}</h3>
             serverSide: true,
             ajax: "{{ url_for('pending_data', status='approved', event_id=event.id) }}",
             deferRender: true,
-            searching: false,
+            searching: true,
             columnDefs: [ {
                 targets:   8,
                 orderable: false,
+                searchable: false,
                 "data": null,
                 "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
                     var arr = oData[8].split(",");

app/templates/feed_config.html

@@ -1,9 +1,101 @@
 {% extends "base.html" %}
 {% block content %}
+
+    {% include "modals/form_config_edit.html" %}
+
     <div class="row">
-        <div class="col-md-12">
-            <h1 class="lead">Current Feed Config</h1>
-            <pre>{{ data }}</pre>
-        </div>
+            <div class="col-md-12">
+                <h1 class="lead">{{ title }}</h1>
+                <button type="button" class="btn btn-primary" id="addButton" data-toggle="modal" data-target="#ConfigEditModal">Add New</button>
+                <table id="config_table" class="table table-bordered">
+                    <thead>
+                        <tr>
+                            <th>Name</th>
+                            <th>Frequency</th>
+                            <th>Event</th>
+                            <th>Config</th>
+                            <th>Actions</th>
+                        </tr>
+                    </thead>
+                    {% for dt in data %}
+                        <tr>
+                            <th scope="row" data-id="{{ loop.index0 }}">{{ dt['name'] }}</th>
+                            <td>{{ dt['frequency'] }}</td>
+                            <td data-id="{{ dt['event_id'] }}">{{ dt['event_name'] }}</td>
+                            <td><pre>{{ dt['modules'] }}</pre></td>
+                            <td data-tablename="data_type">
+                                <button type="button" id="editRow" class="btn btn-primary" data-toggle="modal" data-target="#ConfigEditModal">Edit</button>
+                                <button type="button" id="delRow" class="btn btn-primary" data-toggle="modal" data-target="#ConfigEditModal">Delete</button>
+                            </td>
+                        </tr>
+                    {% endfor %}
+                </table>
+            </div>
     </div>
 {% endblock %}
+
+{% block extra_javascripts %}
+    <script type="text/javascript">
+
+        $(document).delegate('#addButton', 'click', function(){
+            $('.modal-title').text('Edit Config')
+            $('#modal_form').attr('action', '/feeds/config/add')
+
+            $('#name').prop("readonly", false);
+            $('#event').prop("readonly", false);
+            $('#module').prop("readonly", false);
+            $('#frequency').prop("readonly", false);
+            $('#name').val('');
+            $('#frequency').val('*');
+            $('#module').val('');
+        });
+
+
+        $(document).delegate('#editRow', 'click', function(){
+            var row_id = $(this).closest('tr').find('th').eq(0).data('id');
+            var row_name = $(this).closest('tr').find('th').eq(0).text();
+            var row_freq = $(this).closest('tr').find('td').eq(0).text();
+            var row_evt_id = $(this).closest('tr').find('td').eq(1).data('id');
+            var row_evt_name = $(this).closest('tr').find('td').eq(1).text();
+            var row_module = $(this).closest('tr').find('td').eq(2).text();
+
+            $('.modal-title').text('Edit Config')
+            $('#modal_form').attr('action', '/feeds/config/edit')
+
+            $('#name').prop("readonly", false);
+            $('#event').prop("readonly", false);
+            $('#module').prop("readonly", false);
+            $('#frequency').prop("readonly", false);
+
+            $('#index_id').val(row_id);
+            $('#name').val(row_name);
+            $('#frequency').val(row_freq);
+            $('#module').val(row_module);
+            $('#event').val(row_evt_id);
+        });
+
+        $(document).delegate('#delRow', 'click', function(){
+            var row_id = $(this).closest('tr').find('th').eq(0).data('id');
+            var row_name = $(this).closest('tr').find('th').eq(0).text();
+            var row_freq = $(this).closest('tr').find('td').eq(0).text();
+            var row_evt_id = $(this).closest('tr').find('td').eq(1).data('id');
+            var row_evt_name = $(this).closest('tr').find('td').eq(1).text();
+            var row_module = $(this).closest('tr').find('td').eq(2).text();
+
+            $('.modal-title').text('Delete Config')
+            $('#modal_form').attr('action', '/feeds/config/del')
+
+            $('#name').prop("readonly", true);
+            $('#event').prop("readonly", true);
+            $('#module').prop("readonly", true);
+            $('#frequency').prop("readonly", true);
+
+            $('#index_id').val(row_id);
+            $('#name').val(row_name);
+            $('#frequency').val(row_freq);
+            $('#module').val(row_module);
+            $('#event').val(row_evt_id);
+
+        });
+    </script>
+{% endblock %}

app/templates/modals/form_config_add.html

@@ -0,0 +1,60 @@
+{% import "macros/form.html" as forms %}
+    <div class="modal fade" id="myEvModal" tabindex="-1" role="dialog">
+      <div class="modal-dialog" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+            <h4 class="modal-title">Edit Event</h4>
+          </div>
+          <div class="modal-body">
+            <form method="post" action="/event/view/{{ event.id }}" name="event">
+                <div class="form-group row">
+                    <div class="col-md-10">
+                        {{ forms.render_field(ev_form.name) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-12">
+                        {{ forms.render_field(ev_form.details) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.status) }}
+                    </div>
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.source) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.impact) }}
+                    </div>
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.likelihood) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.confidence) }}
+                    </div>
+                    <div class="col-md-6">
+                        {{ forms.render_field(ev_form.tlp) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-6">
+                        <button type="submit" class="btn btn-primary">Save changes</button>
+                    </div>
+                </div>
+                {{ ev_form.hidden_tag() }}
+            </form>
+          </div>
+          <div class="modal-footer">
+            <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
+          </div>
+        </div><!-- /.modal-content -->
+      </div><!-- /.modal-dialog -->
+    </div><!-- /.modal -->

app/templates/modals/form_config_edit.html

@@ -0,0 +1,47 @@
+{% import "macros/form.html" as forms %}
+    <div class="modal fade" id="ConfigEditModal" tabindex="-1" role="dialog">
+      <div class="modal-dialog" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
+            <h4 class="modal-title">Edit Config</h4>
+          </div>
+          <div class="modal-body">
+            <form method="post" action="/feeds/config/" name="modal_form" id="modal_form">
+                <div class="form-group row">
+                    <div class="col-md-10">
+                        {{ forms.render_field(form_edit.name) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-10">
+                        {{ forms.render_field(form_edit.event) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-10">
+                        {{ forms.render_field(form_edit.frequency) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-10">
+                        {{ forms.render_field(form_edit.module) }}
+                    </div>
+                </div>
+                <div class="form-group row">
+                    <div class="col-md-6">
+                        <button type="submit" class="btn btn-primary">Save changes</button>
+                    </div>
+                </div>
+                {{ form_edit.index_id }}
+                {{ form_edit.hidden_tag() }}
+            </form>
+          </div>
+          <div class="modal-footer">
+            <button type="button" class="btn btn-secondary" data-dismiss="modal">Cancel</button>
+          </div>
+        </div><!-- /.modal-content -->
+      </div><!-- /.modal-dialog -->
+    </div><!-- /.modal -->

app/utils.py

@@ -1,7 +1,9 @@
 from app import db
 from app import app
-from .models import Indicator, Control, Itype, Links
+from .models import Indicator, Control, Itype, Links, Event
 from feeder.logentry import ResultsDict
+from whois import whois
+from ipwhois import IPWhois
 import re
 
 
@@ -37,11 +39,20 @@ def _correlate(indicator_list):
                 db.session.add(link2)
     db.session.commit()
 
-def _enrich_data(data):
-    results = None
-    if data['pending']:
-        #impliment
-        return "Not Implemented Yet"
+
+def _enrich_data(data_type, data, pend=True):
+    results = 'Not implemented yet'
+    if pend:
+        if data_type == 'ipv4':
+            obj = IPWhois(data)
+            q = obj.lookup_rdap(depth=1)
+            net = q.get('network', {})
+            results = '%s|%s' % (net.get('name'), net.get('cidr'))
+        elif data_type == 'domain':
+            q = whois(data)
+            results = '%s|%s|%s' % (q.get('registrar'), q.get('name'), q.get('emails'))
+
+    return results
 
 def _valid_json(fields, data_dict):
     if all(k in data_dict for k in fields):
@@ -55,13 +66,20 @@ def _valid_json(fields, data_dict):
 
     return False
 
-def _add_indicators(results, pending=False):
+
+def _add_indicators(results, pending=False, enrich_it=False):
     reasons = []
     inserted_indicators = []
     failed_indicators = []
     updated_indicators = []
     if not isinstance(results, ResultsDict):
         app.logger.warn('Bad object passed to _add_indicators')
+        reasons.append('Bad object passed to _add_indicators')
+        return {'success':len(inserted_indicators), 'failed':len(failed_indicators), 'reason':';'.join(reasons)}
+
+    if not Event.query.get(results.event_id):
+        app.logger.warn('Event ID %s doesnt exist' % results.event_id)
+        reasons.append('Event ID %s doesnt exist' % results.event_id)
         return {'success':len(inserted_indicators), 'failed':len(failed_indicators), 'reason':';'.join(reasons)}
 
     ioc_list, cont_obj, all_data_types = _load_related_data(results)
@@ -88,7 +106,8 @@ def _add_indicators(results, pending=False):
                 updated_indicators.append([ind_id, results.event_id, val])
             else:
                 if (regex and regex.match(val)) or regex is None:
-                    ind = Indicator(results.event_id, val, desc, cont_obj, type_obj, pending, 'Not Processed')
+                    enrich = _enrich_data(data_type, val, pending|enrich_it)
+                    ind = Indicator(results.event_id, val, desc, cont_obj, type_obj, pending, enrich)
                     db.session.add(ind)
                     db.session.flush()
                     ind_id = ind.id