From 4dd3efe25b556ec3d974530b55bcaee600c3efc0 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 9 Aug 2020 18:11:50 +0200 Subject: [PATCH 1/7] disabling usage of frontPeerCache for front peer selection to enable this again, a better selection criteria must be implemented --- src/org/loklak/data/DAO.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/loklak/data/DAO.java b/src/org/loklak/data/DAO.java index 41921c2d1..eb352e5b7 100644 --- a/src/org/loklak/data/DAO.java +++ b/src/org/loklak/data/DAO.java @@ -1550,6 +1550,7 @@ public static ArrayList getFrontPeers() { for (String peer: remote) testpeers.add(peer); return testpeers; } + /* if (frontPeerCache.size() == 0) { // add dynamically all peers that contacted myself for (Map hmap: RemoteAccess.history.values()) { @@ -1559,6 +1560,7 @@ public static ArrayList getFrontPeers() { } } testpeers.addAll(frontPeerCache); + */ return getBestPeers(testpeers); } From 69585536855146d5e8f2ee7a6f8fa0ba9c104593 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 9 Aug 2020 18:38:16 +0200 Subject: [PATCH 2/7] fixed suggestion harvester push --- src/org/loklak/Caretaker.java | 44 ++++++++++--------- .../loklak/harvester/TwitterHarvester.java | 1 + src/org/loklak/http/ClientConnection.java | 27 +++++------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/src/org/loklak/Caretaker.java b/src/org/loklak/Caretaker.java index 73d1d272a..c28c42cdc 100644 --- a/src/org/loklak/Caretaker.java +++ b/src/org/loklak/Caretaker.java @@ -165,32 +165,36 @@ public void run() { try { pendingQueries = LoklakServer.harvester.get_harvest_queries(); } catch (IOException e) { + DAO.severe("FAIL SUGGESTIONS cannot get queries from backend: " + e.getMessage(), e); + break hloop; } - // in case we have queries - if (pendingQueries > 0) { - Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)]; - final AtomicInteger acccount = new AtomicInteger(0); - for (int j = 0; j < rts.length; j++) { - rts[j] = new Thread() { - public void run() { - TwitterTimeline tl = LoklakServer.harvester.harvest_timeline(); - if (tl != null && tl.getQuery() != null) { - /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl); - } - int count = tl == null ? 0 : tl.size(); - acccount.addAndGet(count); + // without queries we cannot do harvesting + if (pendingQueries == 0) break hloop; + + // start harvesting + Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)]; + final AtomicInteger acccount = new AtomicInteger(0); + for (int j = 0; j < rts.length; j++) { + rts[j] = new Thread() { + public void run() { + TwitterTimeline tl = LoklakServer.harvester.harvest_timeline(); + if (tl != null && tl.getQuery() != null) { + /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl); } - }; - rts[j].start(); - try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {} - } - for (Thread t: rts) t.join(); - if (acccount.get() < 0) break hloop; + int count = tl == null ? 0 : tl.size(); + acccount.addAndGet(count); + } + }; + rts[j].start(); try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {} } + for (Thread t: rts) t.join(); + if (acccount.get() < 0) break hloop; + try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {} + + busy = true; } - busy = true; } // run some crawl steps diff --git a/src/org/loklak/harvester/TwitterHarvester.java b/src/org/loklak/harvester/TwitterHarvester.java index 0d77ba767..37e1ed470 100644 --- a/src/org/loklak/harvester/TwitterHarvester.java +++ b/src/org/loklak/harvester/TwitterHarvester.java @@ -120,6 +120,7 @@ public TwitterTimeline harvest_timeline() { if (tl != null && tl.size() > 0) { // find content query strings and store them in the context cache + tl.setQuery(q); checkContext(tl, true); } diff --git a/src/org/loklak/http/ClientConnection.java b/src/org/loklak/http/ClientConnection.java index 8a713e045..b1e33df8a 100644 --- a/src/org/loklak/http/ClientConnection.java +++ b/src/org/loklak/http/ClientConnection.java @@ -421,24 +421,21 @@ public static void download(String source_url, File target_file) { } public static byte[] download(String source_url) throws IOException { + ClientConnection connection = new ClientConnection(source_url, ""); + if (connection.inputStream == null) return null; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int count; + byte[] buffer = new byte[4096]; + IOException ee = null; try { - ClientConnection connection = new ClientConnection(source_url, ""); - if (connection.inputStream == null) return null; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int count; - byte[] buffer = new byte[4096]; - try { - while ((count = connection.inputStream.read(buffer)) > 0) baos.write(buffer, 0, count); - } catch (IOException e) { - DAO.severe(e); - } finally { - connection.close(); - } - return baos.toByteArray(); + while ((count = connection.inputStream.read(buffer)) > 0) baos.write(buffer, 0, count); } catch (IOException e) { - DAO.severe(e); - return null; + ee = e; + } finally { + connection.close(); } + if (ee != null) throw ee; + return baos.toByteArray(); } public int getStatusCode() { From 2e45983be8aff9388840e3fbe82cd4ac1bd212a2 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sun, 9 Aug 2020 18:38:29 +0200 Subject: [PATCH 3/7] missed --- src/org/loklak/Caretaker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/loklak/Caretaker.java b/src/org/loklak/Caretaker.java index c28c42cdc..02d262020 100644 --- a/src/org/loklak/Caretaker.java +++ b/src/org/loklak/Caretaker.java @@ -107,7 +107,7 @@ public void run() { if (SuggestServlet.cache.size() > 100) SuggestServlet.cache.clear(); // sleep a bit to prevent that the DoS limit fires at backend server - try {Thread.sleep(busy ? 500 : 5000);} catch (InterruptedException e) {} + try {Thread.sleep(busy ? 500 : 10000);} catch (InterruptedException e) {} if (!this.shallRun) break beat; busy = false; From 4fad13078d0976c423823ac35db674ce7927f298 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 10 Aug 2020 23:42:24 +0200 Subject: [PATCH 4/7] upgrade shall not fail in case of local changes --- bin/upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/upgrade.sh b/bin/upgrade.sh index 0053f7218..4fa1e509c 100755 --- a/bin/upgrade.sh +++ b/bin/upgrade.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh cd `dirname $0`/.. echo "loading latest code changes" -git pull origin master +git pull -r echo "clean up" ./gradlew clean echo "building loklak" From 440c34d95a4c0bbc1e251685c4b3b9c0f6f6ff4b Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 11 Aug 2020 23:16:16 +0200 Subject: [PATCH 5/7] removed some susi remainings --- src/org/loklak/LoklakServer.java | 2 - src/org/loklak/api/search/ConsoleService.java | 271 ------------------ src/org/loklak/objects/TwitterTimeline.java | 4 - src/org/loklak/susi/SusiProcedures.java | 71 ----- src/org/loklak/susi/SusiThought.java | 98 +------ src/org/loklak/susi/SusiTransfer.java | 201 ------------- 6 files changed, 1 insertion(+), 646 deletions(-) delete mode 100644 src/org/loklak/api/search/ConsoleService.java delete mode 100644 src/org/loklak/susi/SusiProcedures.java delete mode 100644 src/org/loklak/susi/SusiTransfer.java diff --git a/src/org/loklak/LoklakServer.java b/src/org/loklak/LoklakServer.java index 289b3e18f..b9c49136c 100644 --- a/src/org/loklak/LoklakServer.java +++ b/src/org/loklak/LoklakServer.java @@ -104,7 +104,6 @@ import org.loklak.api.p2p.HelloService; import org.loklak.api.p2p.PeersServlet; import org.loklak.api.p2p.PushServlet; -import org.loklak.api.search.ConsoleService; import org.loklak.api.search.EventBriteCrawlerService; import org.loklak.api.search.GenericScraper; import org.loklak.api.search.GithubProfileScraper; @@ -607,7 +606,6 @@ private static void setServerHandler(File dataFile){ HelloService.class, // search - ConsoleService.class, EventBriteCrawlerService.class, MeetupsCrawlerService.class, RSSReaderService.class, diff --git a/src/org/loklak/api/search/ConsoleService.java b/src/org/loklak/api/search/ConsoleService.java deleted file mode 100644 index 7b88e4c07..000000000 --- a/src/org/loklak/api/search/ConsoleService.java +++ /dev/null @@ -1,271 +0,0 @@ -/** - * ConsoleService - * Copyright 13.06.2015 by Michael Peter Christen, @0rb1t3r - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package org.loklak.api.search; - -import java.util.Date; -import java.util.Set; -import java.util.regex.Pattern; - -import org.elasticsearch.search.sort.SortOrder; -import org.json.JSONArray; -import org.json.JSONObject; -import org.loklak.api.cms.TwitterAnalysisService; -import org.loklak.data.DAO; -import org.loklak.geo.GeoMark; -import org.loklak.objects.AccountEntry; -import org.loklak.objects.BasicTimeline.Order; -import org.loklak.objects.QueryEntry; -import org.loklak.objects.ResultList; -import org.loklak.objects.UserEntry; -import org.loklak.server.APIException; -import org.loklak.server.APIHandler; -import org.loklak.server.BaseUserRole; -import org.loklak.server.AbstractAPIHandler; -import org.loklak.server.Authorization; -import org.loklak.server.Query; -import org.loklak.susi.SusiProcedures; -import org.loklak.susi.SusiThought; -import org.loklak.susi.SusiTransfer; -import org.loklak.harvester.BaseScraper; -import org.loklak.harvester.Post; - -import org.loklak.tools.storage.JSONObjectWithDefault; - -import javax.servlet.http.HttpServletResponse; - -/* examples: - * http://localhost:9000/api/console.json?q=SELECT%20text,%20screen_name,%20user.name%20AS%20user%20FROM%20messages%20WHERE%20query=%271%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20messages%20WHERE%20id=%27742384468560912386%27; - * http://localhost:9000/api/console.json?q=SELECT%20link,screen_name%20FROM%20messages%20WHERE%20id=%27742384468560912386%27; - * http://localhost:9000/api/console.json?q=SELECT%20COUNT(*)%20AS%20count,%20screen_name%20AS%20twitterer%20FROM%20messages%20WHERE%20query=%27loklak%27%20GROUP%20BY%20screen_name; - * http://localhost:9000/api/console.json?q=SELECT%20PERCENT(count)%20AS%20percent,%20screen_name%20FROM%20(SELECT%20COUNT(*)%20AS%20count,%20screen_name%20FROM%20messages%20WHERE%20query=%27loklak%27%20GROUP%20BY%20screen_name)%20WHERE%20screen_name%20IN%20(%27leonmakk%27,%27Daminisatya%27,%27sudheesh001%27,%27shiven_mian%27); - * http://localhost:9000/api/console.json?q=SELECT%20query,%20query_count%20AS%20count%20FROM%20queries%20WHERE%20query=%27auto%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20users%20WHERE%20screen_name=%270rb1t3r%27; - * http://localhost:9000/api/console.json?q=SELECT%20place[0]%20AS%20place,%20population,%20location[0]%20AS%20lon,%20location[1]%20AS%20lat%20FROM%20locations%20WHERE%20location=%27Berlin%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20locations%20WHERE%20location=%2753.1,13.1%27; - * http://localhost:9000/api/console.json?q=SELECT%20description%20FROM%20wikidata%20WHERE%20query=%27football%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20meetup%20WHERE%20url=%27http://www.meetup.com/Women-Who-Code-Delhi%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20rss%20WHERE%20url=%27https://www.reddit.com/search.rss?q=loklak%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20eventbrite%20WHERE%20url=%27https://www.eventbrite.fr/e/billets-europeade-2016-concert-de-musique-vocale-25592599153?aff=es2%27; - * http://localhost:9000/api/console.json?q=SELECT%20definition,example%20FROM%20urbandictionary%20WHERE%20query=%27football%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20wordpress%20WHERE%20url=%27https://jigyasagrover.wordpress.com/%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20timeanddate; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20githubProfile%20WHERE%20profile=%27torvalds%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20locationwisetime%20WHERE%20query=%27london%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20instagramprofile%20WHERE%20profile=%27justinpjtrudeau%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20wikigeodata%20WHERE%20place=%27Singapore%27; - * http://localhost:9000/api/console.json?q=SELECT%20*%20FROM%20quoraprofile%20WHERE%20profile=%27justinpjtrudeau%27; - -* */ - -public class ConsoleService extends AbstractAPIHandler implements APIHandler { - - private static final long serialVersionUID = 8578478303032749879L; - public final static SusiProcedures dbAccess = new SusiProcedures(); - - @Override - public BaseUserRole getMinimalBaseUserRole() { return BaseUserRole.ANONYMOUS; } - - @Override - public JSONObject getDefaultPermissions(BaseUserRole baseUserRole) { - return null; - } - - public String getAPIPath() { - return "/api/console.json"; - } - - static { - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?\\( ??SELECT +?(.*?) ??\\) +?WHERE +?(.*?) ?+IN ?+\\((.*?)\\) ??;"), matcher -> { - String subquery = matcher.group(2).trim(); - if (!subquery.endsWith(";")) subquery = subquery + ";"; - String filter_name = matcher.group(3); - JSONArray a0 = dbAccess.deduce("SELECT " + subquery).getJSONArray("data"); - JSONArray a1 = new JSONArray(); - Set filter_set = new SusiTransfer(matcher.group(4)).keys(); - a0.forEach(o -> { - JSONObject j = (JSONObject) o; - if (j.has(filter_name) && filter_set.contains(j.getString(filter_name))) a1.put(j); - }); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return new SusiThought() - .setOffset(0).setHits(a0.length()) - .setData(transfer.conclude(a1)); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?messages +?WHERE +?id ??= ??'([^']*?)' ??;"), matcher -> { - JSONObject message = DAO.messages.readJSON(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return message == null ? null : new SusiThought() - .setOffset(0).setHits(1) - .setData((new JSONArray()).put(transfer.extract(message))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?messages +?WHERE +?query ??= ??'([^']*?)' +?GROUP +?BY +?(.*?) *?;"), matcher -> { - String group = matcher.group(3); - DAO.SearchLocalMessages messages = new DAO.SearchLocalMessages(matcher.group(2), Order.CREATED_AT, 0, 0, 100, group); - JSONArray array = new JSONArray(); - JSONObject aggregation = messages.getAggregations().getJSONObject(group); - - for (String key: aggregation.keySet()) array.put(new JSONObject(true).put(group, key).put("COUNT(*)", aggregation.get(key))); - SusiThought json = messages.timeline.toSusi(true); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(array)); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?messages +?WHERE +?query ??= ??'([^']*?)' ??;"), matcher -> { - DAO.SearchLocalMessages messages = new DAO.SearchLocalMessages(matcher.group(2), Order.CREATED_AT, 0, 100, 0); - SusiThought json = messages.timeline.toSusi(true); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?messages +?WHERE +?query ??= ??'([^']*?)' +?ORDER BY (.*?) ??;"), matcher -> { - DAO.SearchLocalMessages messages = new DAO.SearchLocalMessages(matcher.group(2), Order.valueOf(matcher.group(3)), 0, 100, 0); - SusiThought json = messages.timeline.toSusi(true); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?queries +?WHERE +?query ??= ??'([^']*?)' ??;"), matcher -> { - ResultList queries = DAO.SearchLocalQueries(matcher.group(2), 100, "retrieval_next", "date", SortOrder.ASC, null, new Date(), "retrieval_next"); - SusiThought json = queries.toSusi(); - json.setQuery(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?users +?WHERE +?screen_name ??= ??'([^']*?)' ??;"), matcher -> { - UserEntry user_entry = DAO.searchLocalUserByScreenName(matcher.group(2)); - SusiThought json = new SusiThought(); - json.setQuery(matcher.group(2)); - if (user_entry == null) { - json.setHits(0).setData(new JSONArray()); - } else { - json.setHits(1).setData(new JSONArray().put(user_entry.toJSON())); - } - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?accounts +?WHERE +?screen_name ??= ??'(.*?)' ??;"), matcher -> { - AccountEntry account_entry = DAO.searchLocalAccount(matcher.group(2)); - SusiThought json = new SusiThought(); - json.setQuery(matcher.group(2)); - if (account_entry == null) { - json.setHits(0).setData(new JSONArray()); - } else { - json.setHits(1).setData(new JSONArray().put(account_entry.toJSON())); - } - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?locations +?WHERE +?location ??= ??'(.*?)' ??;"), matcher -> { - GeoMark loc = DAO.geoNames.analyse(matcher.group(2), null, 5, Long.toString(System.currentTimeMillis())); - SusiThought json = new SusiThought(); - json.setQuery(matcher.group(2)); - if (loc == null) { - json.setHits(0).setData(new JSONArray()); - } else { - json.setHits(1).setData(new JSONArray().put(loc.toJSON(false))); - } - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - return json.setData(transfer.conclude(json.getJSONArray("data"))); - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?meetup +?WHERE +?url ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = MeetupsCrawlerService.crawlMeetups(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?rss +?WHERE +?url ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = RSSReaderService.readRSS(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?eventbrite +?WHERE +?url ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = EventBriteCrawlerService.crawlEventBrite(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?wordpress +?WHERE +?url ??= ??'(.*?)' ??;"), matcher -> { - BaseScraper wordpressScrape = new WordpressCrawlerService(matcher.group(2)); - SusiThought json = new SusiThought(wordpressScrape.getData()); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?timeanddate;"), matcher -> { - SusiThought json = TimeAndDateService.timeAndDate(); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?githubProfile +?WHERE +?profile ??= ??'(.*?)' ??;"), matcher -> { - BaseScraper githubScrape = new GithubProfileScraper(matcher.group(2)); - Post postObj = githubScrape.getData(); - postObj.put("data", postObj.get("user")).remove("user"); - SusiThought json = new SusiThought(postObj); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?locationwisetime +?WHERE +?query ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = LocationWiseTimeService.locationWiseTime(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?twitanalysis +?WHERE +?screen_name ??= ??'(.*?)' +?AND +?count ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = TwitterAnalysisService.showAnalysis(matcher.group(2), matcher.group(3)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?instagramprofile +?WHERE +?profile ??= ??'(.*?)' ??;"), matcher -> { - BaseScraper instaScrape = new InstagramProfileScraper(matcher.group(2)); - SusiThought json = new SusiThought(instaScrape.getData()); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?quoraprofile +?WHERE +?profile ??= ??'(.*?)' ??;"), matcher -> { - BaseScraper quoraScrape = new QuoraProfileScraper(matcher.group(2)); - SusiThought json = new SusiThought(quoraScrape.getData()); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - - dbAccess.put(Pattern.compile("SELECT +?(.*?) +?FROM +?wikigeodata +?WHERE +?place ??= ??'(.*?)' ??;"), matcher -> { - SusiThought json = WikiGeoData.wikiGeoData(matcher.group(2)); - SusiTransfer transfer = new SusiTransfer(matcher.group(1)); - json.setData(transfer.conclude(json.getData())); - return json; - }); - } - - @Override - public JSONObject serviceImpl(Query post, HttpServletResponse response, Authorization rights, final JSONObjectWithDefault permissions) throws APIException { - - // parameters - String q = post.get("q", ""); - //int timezoneOffset = post.get("timezoneOffset", 0); - - return dbAccess.deduce(q); - } - -} diff --git a/src/org/loklak/objects/TwitterTimeline.java b/src/org/loklak/objects/TwitterTimeline.java index 3f320c7a4..48f2c04cd 100644 --- a/src/org/loklak/objects/TwitterTimeline.java +++ b/src/org/loklak/objects/TwitterTimeline.java @@ -131,10 +131,6 @@ public JSONObject toJSON(boolean withEnrichedData, String metadata_field_name, S return json; } - public SusiThought toSusi(boolean withEnrichedData) throws JSONException { - return toSusi(withEnrichedData, new SusiThought()); - } - private SusiThought toSusi(boolean withEnrichedData, SusiThought json) throws JSONException { json .setQuery(this.query) diff --git a/src/org/loklak/susi/SusiProcedures.java b/src/org/loklak/susi/SusiProcedures.java deleted file mode 100644 index cb463d422..000000000 --- a/src/org/loklak/susi/SusiProcedures.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * SusiIntent - * Copyright 14.07.2016 by Michael Peter Christen, @0rb1t3r - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package org.loklak.susi; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class SusiProcedures extends LinkedHashMap> implements Map> { - - private static final long serialVersionUID = 4531596762427825563L; - - public SusiProcedures() { - super(); - } - - /** - * Deduction is the application of an intent on perception and a world model. - * In this method the mappings from the intent set is applied to the perception q and previous - * deduction steps as given with the flow. Every mapping that has a matcher - * with the perception causes the application of the stored lambda function on the perception - * producing a thought. If the thought generation is not successful (which means that the lambda - * fails or produces a null output) then the next mappings from the intent set is tried. - * In case that no inspiration is possible, an empty thought is produced, containing nothing. - * @param q the perception - * @return a thought from the application of the intent set - */ - public SusiThought deduce(String q) { - if (q == null) return new SusiThought(); - q = q.trim(); - for (Map.Entry> pe: this.entrySet()) { - Pattern p = pe.getKey(); - Matcher m = p.matcher(q); - if (m.find()) try { - SusiThought json = pe.getValue().apply(m); - if (json != null) { - json.setProcess(p.pattern()); - return json; - } - } catch (Throwable e) { - // applying a intent may produce various failure, including - // - IOExceptions if the intent needs external resources - // - NullPointerException if the intent needs a flow which can be null in case of the attempt of an inspiration - // we silently ignore these exceptions as they are normal and acceptable during thinking - } - } - - // no success: produce an empty thought - return new SusiThought(); - } - -} diff --git a/src/org/loklak/susi/SusiThought.java b/src/org/loklak/susi/SusiThought.java index 08ca4ee08..1460d8e51 100644 --- a/src/org/loklak/susi/SusiThought.java +++ b/src/org/loklak/susi/SusiThought.java @@ -20,9 +20,6 @@ package org.loklak.susi; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - import org.json.JSONArray; import org.json.JSONObject; @@ -44,37 +41,6 @@ public SusiThought() { this.data_name = "data"; } - /** - * create a clone of a json object as a SusiThought object - * @param json the 'other' thought, probably an exported and re-imported thought - */ - public SusiThought(JSONObject json) { - this(); - if (json.has(this.metadata_name)) this.put(this.metadata_name, json.getJSONObject(this.metadata_name)); - if (json.has(this.data_name)) this.setData(json.getJSONArray(this.data_name)); - if (json.has("actions")) this.put("actions", json.getJSONArray("actions")); - } - - /** - * Create an initial thought using the matcher on an expression. - * Such an expression is like the input from a text source which contains keywords - * that are essential for the thought. The matcher extracts such information. - * Matching informations are named using the order of the appearance of the information pieces. - * The first information is named '1', the second '2' and so on. The whole information which contained - * the matching information is named '0'. - * @param matcher - */ - public SusiThought(Matcher matcher) { - this(); - this.setOffset(0).setHits(1); - JSONObject row = new JSONObject(); - row.put("0", matcher.group(0)); - for (int i = 0; i < matcher.groupCount(); i++) { - row.put(Integer.toString(i + 1), matcher.group(i + 1)); - } - this.setData(new JSONArray().put(row)); - } - @Deprecated public SusiThought(String metadata_name, String data_name) { super(true); @@ -202,67 +168,5 @@ public JSONArray getData() { this.put(data_name, a); return a; } - - /** - * Merging of data is required during an mind-meld. - * To meld two thoughts, we combine their data arrays into one. - * The resulting table has the maximum length of the source tables - * @param table the information to be melted into our existing table. - * @return the thought - */ - public SusiThought mergeData(JSONArray table1) { - JSONArray table0 = this.getData(); - while (table0.length() < table1.length()) table0.put(new JSONObject()); - for (int i = 0; i < table1.length(); i++) { - table0.getJSONObject(i).putAll(table1.getJSONObject(i)); - } - setData(table0); - return this; - } - - /** - * If during thinking we observe something that we want to memorize, we can memorize this here - * @param featureName the object key - * @param observation the object value - * @return the thought - */ - public SusiThought addObservation(String featureName, String observation) { - JSONArray data = getData(); - for (int i = 0; i < data.length(); i++) { - JSONObject spark = data.getJSONObject(i); - if (!spark.has(featureName)) { - spark.put(featureName, observation); - return this; - } - } - data.put(new JSONObject().put(featureName, observation)); - return this; - } - - public static final Pattern variable_pattern = Pattern.compile("\\$.*?\\$"); - - /** - * Unification applies a piece of memory within the current argument to a statement - * which creates an instantiated statement - * @param statement - * @return the instantiated statement with elements of the argument applied as much as possible - */ - public String unify(String statement) { - JSONArray table = this.getData(); - if (table != null && table.length() > 0) { - JSONObject row = table.getJSONObject(0); - for (String key: row.keySet()) { - int i = statement.indexOf("$" + key + "$"); - if (i >= 0) { - statement = statement.substring(0, i) + row.get(key).toString() + statement.substring(i + key.length() + 2); - } - } - } - return statement; - } - - public static void main(String[] args) { - SusiThought t = new SusiThought().addObservation("a", "letter-a"); - System.out.println(t.unify("the letter $a$")); - } + } diff --git a/src/org/loklak/susi/SusiTransfer.java b/src/org/loklak/susi/SusiTransfer.java deleted file mode 100644 index f5693aceb..000000000 --- a/src/org/loklak/susi/SusiTransfer.java +++ /dev/null @@ -1,201 +0,0 @@ -/** - * SusiTransfer - * Copyright 14.07.2016 by Michael Peter Christen, @0rb1t3r - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program in the file lgpl21.txt - * If not, see . - */ - -package org.loklak.susi; - -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Set; - -import org.json.JSONArray; -import org.json.JSONObject; - -import com.google.common.util.concurrent.AtomicDouble; - -/** - * Transfer is the ability to perceive a given thought in a different representation - * in such a way that it applies on an intent or an intent set. - */ -public class SusiTransfer { - - private LinkedHashMap selectionMapping; - - /** - * Create a new transfer. The mapping must be given in the same way as SQL column selection - * statements. The selection of sub-objects of json object can be done using dot-notion. - * Arrays can be accessed using brackets '[' and ']'. An example is: - * mapping = "location.lon AS longitude, location.lat AS latitude" - * or - * mapping = "names[0] AS firstname" - * As a reference, the mappingExpression shall be superset of a list of - * https://mariadb.com/kb/en/mariadb/select/#select-expressions - * @param mapping - */ - public SusiTransfer(String mappingExpression) { - this.selectionMapping = parse(mappingExpression); - } - - /** - * get the set of transfer keys - * @return - */ - public Set keys() { - return this.selectionMapping.keySet(); - } - - /** - * transfer mappings can be used to extract specific information from a json object to - * create a new json object. In the context of Susi this is applied on choices from thought data - * @param choice one 'row' of a SusiThought data array - * @return a choice where the elements of the given choice are extracted according to the given mapping - */ - public JSONObject extract(JSONObject choice) { - if (this.selectionMapping == null) return choice; - JSONObject json = new JSONObject(true); - for (Map.Entry c: selectionMapping.entrySet()) { - String key = c.getKey(); - int p = key.indexOf('.'); - if (p > 0) { - // sub-element - String k0 = key.substring(0, p); - String k1 = key.substring(p + 1); - if (choice.has(k0)) { - if (k1.equals("length") || k1.equals("size()")) { - Object a = choice.get(k0); - if (a instanceof String[]) { - json.put(c.getValue(),((String[]) a).length); - } else if (a instanceof JSONArray) { - json.put(c.getValue(),((JSONArray) a).length()); - } - } else { - JSONObject o = choice.getJSONObject(k0); - if (o.has(k1)) json.put(c.getValue(), o.get(k1)); - } - } - } else if ((p = key.indexOf('[')) > 0) { - // array - int q = key.indexOf("]", p); - if (q > 0) { - String k0 = key.substring(0, p); - int i = Integer.parseInt(key.substring(p + 1, q)); - if (choice.has(k0)) { - JSONArray a = choice.getJSONArray(k0); - if (i < a.length()) json.put(c.getValue(), a.get(i)); - } - } - } else { - // flat - if (choice.has(key)) json.put(c.getValue(), choice.get(key)); - } - } - return json; - } - - /** - * A conclusion from choices is done by the application of a function on the choice set. - * This may be done by i.e. counting the number of choices or extracting a maximum element. - * @param choices the given set of json objects from the data object of a SusiThought - * @returnan array of json objects which are the extraction of given choices according to the given mapping - */ - public JSONArray conclude(JSONArray choices) { - JSONArray a = new JSONArray(); - if (this.selectionMapping != null && this.selectionMapping.size() == 1) { - // test if this has an aggregation key: AVG, COUNT, MAX, MIN, SUM - final String aggregator = this.selectionMapping.keySet().iterator().next(); - final String aggregator_as = this.selectionMapping.get(aggregator); - if (aggregator.startsWith("COUNT(") && aggregator.endsWith(")")) { // TODO: there should be a special pattern for this to make it more efficient - return a.put(new JSONObject().put(aggregator_as, choices.length())); - } - if (aggregator.startsWith("MAX(") && aggregator.endsWith(")")) { - final AtomicDouble max = new AtomicDouble(Double.MIN_VALUE); String c = aggregator.substring(4, aggregator.length() - 1); - choices.forEach(json -> max.set(Math.max(max.get(), ((JSONObject) json).getDouble(c)))); - return a.put(new JSONObject().put(aggregator_as, max.get())); - } - if (aggregator.startsWith("MIN(") && aggregator.endsWith(")")) { - final AtomicDouble min = new AtomicDouble(Double.MAX_VALUE); String c = aggregator.substring(4, aggregator.length() - 1); - choices.forEach(json -> min.set(Math.min(min.get(), ((JSONObject) json).getDouble(c)))); - return a.put(new JSONObject().put(aggregator_as, min.get())); - } - if (aggregator.startsWith("SUM(") && aggregator.endsWith(")")) { - final AtomicDouble sum = new AtomicDouble(0.0d); String c = aggregator.substring(4, aggregator.length() - 1); - choices.forEach(json -> sum.addAndGet(((JSONObject) json).getDouble(c))); - return a.put(new JSONObject().put(aggregator_as, sum.get())); - } - if (aggregator.startsWith("AVG(") && aggregator.endsWith(")")) { - final AtomicDouble sum = new AtomicDouble(0.0d); String c = aggregator.substring(4, aggregator.length() - 1); - choices.forEach(json -> sum.addAndGet(((JSONObject) json).getDouble(c))); - return a.put(new JSONObject().put(aggregator_as, sum.get() / choices.length())); - } - } - if (this.selectionMapping != null && this.selectionMapping.size() == 2) { - Iterator ci = this.selectionMapping.keySet().iterator(); - String aggregator = ci.next(); String column = ci.next(); - if (column.indexOf('(') >= 0) {String s = aggregator; aggregator = column; column = s;} - final String aggregator_as = this.selectionMapping.get(aggregator); - final String column_as = this.selectionMapping.get(column); - final String column_final = column; - if (aggregator.startsWith("PERCENT(") && aggregator.endsWith(")")) { - final AtomicDouble sum = new AtomicDouble(0.0d); String c = aggregator.substring(8, aggregator.length() - 1); - choices.forEach(json -> sum.addAndGet(((JSONObject) json).getDouble(c))); - choices.forEach(json -> a.put(new JSONObject() - .put(aggregator_as, 100.0d * ((JSONObject) json).getDouble(c) / sum.get()) - .put(column_as, ((JSONObject) json).get(column_final)))); - return a; - } - } - for (Object json: choices) { - JSONObject extraction = this.extract((JSONObject) json); - if (extraction.length() > 0) a.put(extraction); - } - return a; - } - - private static LinkedHashMap parse(String mapping) { - LinkedHashMap columns; - String[] column_list = mapping.trim().split(","); - if (column_list.length == 1 && column_list[0].equals("*")) { - columns = null; - } else { - columns = new LinkedHashMap<>(); - for (String column: column_list) { - String c = column.trim(); - int p = c.indexOf(" AS "); - if (p < 0) { - c = trimQuotes(c); - columns.put(c, c); - } else { - columns.put(trimQuotes(c.substring(0, p).trim()), trimQuotes(c.substring(p + 4).trim())); - } - } - } - return columns; - } - - private static String trimQuotes(String s) { - if (s.length() == 0) return s; - if (s.charAt(0) == '\'' || s.charAt(0) == '\"') s = s.substring(1); - if (s.charAt(s.length() - 1) == '\'' || s.charAt(s.length() - 1) == '\"') s = s.substring(0, s.length() - 1); - return s; - } - - public String toString() { - return this.selectionMapping == null ? "NULL" : this.selectionMapping.toString(); - } -} From 1577e039c450dacdbfcfb35c21d0b4b6991d2aca Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Tue, 11 Aug 2020 23:25:59 +0200 Subject: [PATCH 6/7] simplified upgrade --- bin/upgrade.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/upgrade.sh b/bin/upgrade.sh index 4fa1e509c..dc5cd6f7c 100755 --- a/bin/upgrade.sh +++ b/bin/upgrade.sh @@ -2,8 +2,6 @@ cd `dirname $0`/.. echo "loading latest code changes" git pull -r -echo "clean up" -./gradlew clean -echo "building loklak" -./gradlew build +echo "assembling loklak" +./gradlew assemble bin/restart.sh From d866d7db9249af9d2bed15f99e1146795a3bec66 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sat, 22 Aug 2020 13:54:06 +0200 Subject: [PATCH 7/7] bugfixing --- src/org/json/JSONObject.java | 2 +- src/org/loklak/data/DAO.java | 8 ++++++-- src/org/loklak/harvester/TwitterScraper.java | 12 +++++++----- src/org/loklak/objects/AbstractObjectEntry.java | 4 ++-- src/org/loklak/objects/QueryEntry.java | 8 ++++++-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/org/json/JSONObject.java b/src/org/json/JSONObject.java index 0ba3f2112..9ea0ea9a1 100755 --- a/src/org/json/JSONObject.java +++ b/src/org/json/JSONObject.java @@ -297,7 +297,7 @@ public JSONObject(JSONTokener x) throws JSONException { */ public JSONObject(Map m) { if (m == null) { - this.map = new HashMap(); + this.map = new LinkedHashMap(); } else { this.map = new HashMap(m.size()); for (final Entry e : m.entrySet()) { diff --git a/src/org/loklak/data/DAO.java b/src/org/loklak/data/DAO.java index eb352e5b7..f030e9c9b 100644 --- a/src/org/loklak/data/DAO.java +++ b/src/org/loklak/data/DAO.java @@ -1302,7 +1302,11 @@ public static ResultList SearchLocalQueries(final String q, final in ResultList> result = elasticsearch_client.fuzzyquery(IndexName.queries.name(), "query", q, resultCount, sort_field, default_sort_type, sort_order, since, until, range_field); queries.setHits(result.getHits()); for (Map map: result) { - queries.add(new QueryEntry(new JSONObject(map))); + QueryEntry qe = new QueryEntry(new JSONObject(map)); + // check a flag value for queries that probably never get new messages + if (qe.getMessagePeriod() != QueryEntry.DAY_MILLIS) { + queries.add(qe); + } } return queries; } @@ -1415,7 +1419,7 @@ public static TwitterTimeline scrapeTwitter( DAO.severe(e); } - if (recordQuery && Caretaker.acceptQuery4Retrieval(q)) { + if (recordQuery && Caretaker.acceptQuery4Retrieval(q) && tl.size() > 0) { if (qe == null) { // a new query occurred qe = new QueryEntry(q, timezoneOffset, tl.period(), SourceType.TWITTER, byUserQuery); diff --git a/src/org/loklak/harvester/TwitterScraper.java b/src/org/loklak/harvester/TwitterScraper.java index 234401489..b4aaa4765 100644 --- a/src/org/loklak/harvester/TwitterScraper.java +++ b/src/org/loklak/harvester/TwitterScraper.java @@ -218,7 +218,7 @@ private static TwitterTimeline[] search( // parse Elements items = doc.getElementsByClass("stream-item"); - for (int itemc = 0; itemc < items.size(); itemc++) { + itemloop: for (int itemc = 0; itemc < items.size(); itemc++) { Element item = items.get(itemc); if (debuglog) System.out.println(item.toString()); @@ -258,10 +258,12 @@ private static TwitterTimeline[] search( } String tweettimes = timestamp.attr("data-time-ms"); + if (tweettimes.length() == 0) continue itemloop; // sometimes tweets are not available any more long tweettime = Long.parseLong(tweettimes); - long snowflaketime = snowflake2millis(Long.parseLong(tweetID)); - assert tweettime / 1000 == snowflaketime / 1000; - + // this assertion holds in most, but unfortunately not in all cases, so it is commented out + // long snowflaketime = snowflake2millis(Long.parseLong(tweetID)); + // assert tweettime / 1000L == snowflaketime / 1000L : "tweettime = " + tweettime + ", snowflaketime = " + snowflaketime; + Elements reply = item.getElementsByClass("ProfileTweet-action--reply").get(0).children(); Elements retweet = item.getElementsByClass("ProfileTweet-action--retweet").get(0).children(); Elements favourite = item.getElementsByClass("ProfileTweet-action--favorite").get(0).children(); @@ -747,7 +749,7 @@ public Post toJSON(final UserEntry user, final boolean calculatedData, final int // the tweet; the cleanup is a helper function which cleans mistakes from the past in scraping MessageEntry.TextLinkMap tlm = this.moreData.getText(iflinkexceedslength, urlstub, this.text, this.getLinks(), this.getPostId()); this.put("text", tlm); - if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm()); + if (this.status_id_url != null) this.put("link", this.status_id_url.toExternalForm()); // this is the primary key for retrieval in elasticsearch this.put("id_str", this.postId); this.put("conversation_id", this.conversationID); this.put("conversation_user", this.conversationUserIDs); diff --git a/src/org/loklak/objects/AbstractObjectEntry.java b/src/org/loklak/objects/AbstractObjectEntry.java index bb44f3d15..5c5a4791c 100644 --- a/src/org/loklak/objects/AbstractObjectEntry.java +++ b/src/org/loklak/objects/AbstractObjectEntry.java @@ -35,8 +35,8 @@ public abstract class AbstractObjectEntry extends Post implements ObjectEntry { - public final static String TIMESTAMP_FIELDNAME = "timestamp"; - public final static String CREATED_AT_FIELDNAME = "created_at"; + public final static String TIMESTAMP_FIELDNAME = "timestamp"; // the harvesting time, NOT used for identification + public final static String CREATED_AT_FIELDNAME = "created_at"; // the tweet time as embedded in the tweet, not used for identification either public AbstractObjectEntry() { } diff --git a/src/org/loklak/objects/QueryEntry.java b/src/org/loklak/objects/QueryEntry.java index b29f1c633..a7046d7d8 100644 --- a/src/org/loklak/objects/QueryEntry.java +++ b/src/org/loklak/objects/QueryEntry.java @@ -64,7 +64,7 @@ */ public class QueryEntry extends AbstractObjectEntry implements ObjectEntry { - private final static long DAY_MILLIS = 1000L * 60L * 60L * 24L; + public final static long DAY_MILLIS = 1000L * 60L * 60L * 24L; private final static int RETRIEVAL_CONSTANT = 20; // the number of messages that we get with each retrieval at maximum protected String query; // the query in the exact way as the user typed it in @@ -144,7 +144,7 @@ public void update(final long message_period, final boolean byUserQuery) { this.query_last = this.retrieval_last; } long new_message_period = message_period; // can be Long.MAX_VALUE if less than 2 messages are in timeline! - int new_messages_per_day = (int) (DAY_MILLIS / new_message_period); // this is an interpolation based on the last tweet list, can be 0! + int new_messages_per_day = (int) (DAY_MILLIS / (new_message_period + 1)); // this is an interpolation based on the last tweet list, can be 0! if (new_message_period == Long.MAX_VALUE || new_messages_per_day == 0) { this.message_period = DAY_MILLIS; } else { @@ -217,6 +217,10 @@ public int getQueryCount() { public int getRetrievalCount() { return this.retrieval_count; } + + public long getMessagePeriod() { + return this.message_period; + } public int getMessagesPerDay() { return this.messages_per_day;