diff --git a/build.gradle b/build.gradle index f4bed17..c3fedd0 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ buildscript { - ext.kotlin_version = '1.3.61' + ext.kotlin_version = '1.3.71' repositories { mavenCentral() diff --git a/common/build.gradle b/common/build.gradle index dc4bfa6..669a586 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -13,7 +13,7 @@ dependencies { // API api("com.google.protobuf:protobuf-java:3.6.1") api("io.grpc:grpc-stub:1.18.0") - api("de.hpi.cloud:hpi-cloud:0.0.13") + api("de.hpi.cloud:hpi-cloud:0.0.14") // Storage api("com.couchbase.client:java-client:2.7.9") diff --git a/common/src/main/kotlin/de/hpi/cloud/common/couchbase/ApiDocument.kt b/common/src/main/kotlin/de/hpi/cloud/common/couchbase/ApiDocument.kt index 4275674..260e544 100644 --- a/common/src/main/kotlin/de/hpi/cloud/common/couchbase/ApiDocument.kt +++ b/common/src/main/kotlin/de/hpi/cloud/common/couchbase/ApiDocument.kt @@ -9,7 +9,7 @@ import de.hpi.cloud.common.grpc.throwAlreadyExists inline fun , reified Proto : GeneratedMessageV3> Bucket.tryInsert(wrapper: Wrapper) { try { - insert(wrapper) + insert(wrapper) } catch (e: DocumentAlreadyExistsException) { throwAlreadyExists(wrapper.id) } diff --git a/common/src/main/kotlin/de/hpi/cloud/common/couchbase/Document.kt b/common/src/main/kotlin/de/hpi/cloud/common/couchbase/Document.kt index a49737d..d87fa22 100644 --- a/common/src/main/kotlin/de/hpi/cloud/common/couchbase/Document.kt +++ b/common/src/main/kotlin/de/hpi/cloud/common/couchbase/Document.kt @@ -3,7 +3,6 @@ package de.hpi.cloud.common.couchbase import com.couchbase.client.java.AsyncBucket import com.couchbase.client.java.Bucket import com.couchbase.client.java.document.RawJsonDocument -import com.google.protobuf.GeneratedMessageV3 import de.hpi.cloud.common.entity.Entity import de.hpi.cloud.common.entity.Id import de.hpi.cloud.common.entity.Wrapper @@ -37,9 +36,9 @@ inline fun > Wrapper.toJsonDocument(): RawJsonDocument ) } -inline fun , reified Proto : GeneratedMessageV3> Bucket.insert(wrapper: Wrapper) = +inline fun > Bucket.insert(wrapper: Wrapper) = insert(wrapper.toJsonDocument()) -inline fun > Bucket.upsert(entityWrapper: Wrapper) { - upsert(entityWrapper.toJsonDocument()) +inline fun > Bucket.upsert(wrapper: Wrapper) { + upsert(wrapper.toJsonDocument()) } diff --git a/common/src/main/kotlin/de/hpi/cloud/common/entity/Entity.kt b/common/src/main/kotlin/de/hpi/cloud/common/entity/Entity.kt index 4527c8e..14bb2e0 100644 --- a/common/src/main/kotlin/de/hpi/cloud/common/entity/Entity.kt +++ b/common/src/main/kotlin/de/hpi/cloud/common/entity/Entity.kt @@ -3,11 +3,9 @@ package de.hpi.cloud.common.entity import com.google.protobuf.GeneratedMessageV3 import de.hpi.cloud.common.Context import de.hpi.cloud.common.protobuf.setId -import de.hpi.cloud.common.serializers.proto.InstantSerializer import kotlinx.serialization.ImplicitReflectionSerializer import kotlinx.serialization.KSerializer import kotlinx.serialization.serializer -import java.time.Instant import kotlin.reflect.KClass import kotlin.reflect.full.companionObjectInstance import de.hpi.cloud.common.serializers.proto.ProtoSerializer as AnyProtoSerializer diff --git a/service-food-crawler/build.gradle b/service-food-crawler/build.gradle deleted file mode 100644 index 2b8241c..0000000 --- a/service-food-crawler/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.40") - classpath("com.github.jengelman.gradle.plugins:shadow:5.1.0") - } -} - -plugins { - id "kotlin" - id "application" - id "com.github.johnrengelman.shadow" version "5.1.0" -} - -repositories { - jcenter() - maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" } -} - -dependencies { - implementation(project(":common")) - - // Crawling - implementation("com.beust:klaxon:5.0.11") -} - -application { - mainClassName = "de.hpi.cloud.food.crawler.MainKt" -} diff --git a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/Main.kt b/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/Main.kt deleted file mode 100644 index 764b6dc..0000000 --- a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/Main.kt +++ /dev/null @@ -1,44 +0,0 @@ -package de.hpi.cloud.food.crawler - -import de.hpi.cloud.common.utils.couchbase.withBucket -import de.hpi.cloud.food.crawler.canteens.Griebnitzsee -import de.hpi.cloud.food.crawler.canteens.Ulf -import java.time.LocalDate - -const val NAME = "HPI-MobileDev-Crawler[Food]" -const val VERSION = "1.0.0" -const val CRAWLER_INFO = "$NAME/$VERSION" -val USER_AGENT = "$CRAWLER_INFO klaxon/5.0.11 Kotlin-runtime/${KotlinVersion.CURRENT}" - -val KNOWN_CANTEENS = setOf( - Griebnitzsee, - Ulf -) - -fun main(args: Array) { - val todayOnly = args.any { it.equals("--today", true) } - - println("Starting $NAME") - withBucket("food") { bucket -> - // TODO: clear previous data - KNOWN_CANTEENS - .map { OpenMensaCrawler(it) } - .forEach { crawler -> - println("Starting crawler $CRAWLER_INFO - ${crawler.canteenData.id}") - println("Using User-Agent=\"$USER_AGENT\"") - - val days = - if (todayOnly) listOf(LocalDate.now()) - else crawler.queryDays() - .filter { it.isOpen } - .map { it.date } - days.forEach { date -> - print("Date $date: ") - - val meals = crawler.queryMeals(date) - meals.forEach { bucket.upsert(it.toJsonDocument()) } - println("${meals.size} meals") - } - } - } -} diff --git a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaCrawler.kt b/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaCrawler.kt deleted file mode 100644 index 2902117..0000000 --- a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaCrawler.kt +++ /dev/null @@ -1,115 +0,0 @@ -package de.hpi.cloud.food.crawler - -import com.beust.klaxon.Klaxon -import de.hpi.cloud.common.Entity -import de.hpi.cloud.common.utils.couchbase.i18nMap -import de.hpi.cloud.common.utils.protobuf.euros -import de.hpi.cloud.common.utils.protobuf.toDbMap -import de.hpi.cloud.common.utils.protobuf.toTimestamp -import de.hpi.cloud.food.crawler.utils.openStreamWith -import java.net.URI -import java.time.LocalDate -import java.time.format.DateTimeFormatter - -class OpenMensaCrawler( - val canteenData: CanteenData -) { - companion object { - private const val OPENMENSA_API_VERSION = "v2" - - val BASE_URI: URI = URI("https://openmensa.org/api/$OPENMENSA_API_VERSION/") - val DATE_FORMAT_ISO8601 = DateTimeFormatter.ISO_LOCAL_DATE!! - val DATE_FORMAT_COMPACT = DateTimeFormatter.ofPattern("yyyyMMdd")!! - } - - private val klaxon = Klaxon() - - private fun canteenQuery() = "canteens/${canteenData.openMensaId}" - private fun daysQuery() = "${canteenQuery()}/days" - private fun mealsQuery(date: LocalDate) = "${daysQuery()}/${date.format(DATE_FORMAT_ISO8601)}/meals" - - private fun streamJsonApi(query: String) = BASE_URI.resolve(query).toURL() - .openStreamWith( - "User-agent" to USER_AGENT, - "Accept" to "application/json, text/plain" - ) - - fun queryDays() = streamJsonApi(daysQuery()) - .bufferedReader() - .use { reader -> - klaxon.parseJsonArray(reader) - .mapChildrenObjectsOnly { - DateStatus( - date = LocalDate.parse(it.string("date")!!, DATE_FORMAT_ISO8601), - isOpen = !it.boolean("closed")!! - ) - } - .sortedBy { it.date } - } - - - fun queryMeals(date: LocalDate) = streamJsonApi(mealsQuery(date)) - .bufferedReader() - .use { reader -> - klaxon.parseArray(reader)!! - .flatMap { MensaMeal.parseMeals(canteenData, date, it) } - .let { canteenData.mapReduce(it) } - .let { canteenData.deduplicate(it) } - } -} - -data class DateStatus( - val date: LocalDate, - val isOpen: Boolean -) - -data class MensaMeal( - private val canteenData: CanteenData, - val openMensaMeal: OpenMensaMeal, - val date: LocalDate, - val counter: String?, - val title: String, - val offerName: String, - val labelIds: List, - val uniqueIdSuffix: Int? = null -) : Entity("menuItem", 1) { - companion object { - fun parseMeals(canteenData: CanteenData, date: LocalDate, openMensaMeal: OpenMensaMeal) = - MensaMeal( - canteenData, - openMensaMeal, - date, - canteenData.findCounter(openMensaMeal), - openMensaMeal.name.replace("\n", ""), - canteenData.findOfferName(openMensaMeal), - canteenData.findLabels(openMensaMeal) - ).let { listOf(it) } - } - - override val id - get() = canteenData.id + - "-${date.format(OpenMensaCrawler.DATE_FORMAT_COMPACT)}" + - "-${counter ?: openMensaId}" + - (uniqueIdSuffix?.let { "_$it" } ?: "") - private val openMensaId get() = openMensaMeal.id - - private val prices = openMensaMeal.prices - .mapKeys { - if (it.key == "others") "default" - else it.key - } - .mapValues { - it.value.euros().toDbMap() - } - - override fun valueToMap() = mapOf( - "restaurantId" to canteenData.id, - "openMensaId" to openMensaId, - "date" to date.toTimestamp().toDbMap(), - "offerName" to offerName, - "title" to i18nMap(de = title), - "counter" to i18nMap(de = counter), - "labelIds" to labelIds, - "prices" to prices - ) -} diff --git a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaTypes.kt b/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaTypes.kt deleted file mode 100644 index dad1817..0000000 --- a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/OpenMensaTypes.kt +++ /dev/null @@ -1,31 +0,0 @@ -package de.hpi.cloud.food.crawler - -data class OpenMensaMeal( - val id: Int, - val name: String, - val category: String, - val prices: Map, - val notes: List -) - -open class CanteenData( - val id: String, - val openMensaId: Int -) { - open fun findOfferName(meal: OpenMensaMeal): String = meal.category - open fun findLabels(meal: OpenMensaMeal): List = listOf() - open fun findCounter(meal: OpenMensaMeal): String? = null - - open fun mapReduce(meals: List): List = meals - open fun deduplicate(meals: List): List = meals - // meals with the same counter have the same id - let's fix this - .groupBy { it.id } - .flatMap { - if (it.value.size > 1) - it.value.mapIndexed { index, meal -> - meal.copy(uniqueIdSuffix = index + 1) - } - else // no duplicate - it.value - } -} diff --git a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Griebnitzsee.kt b/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Griebnitzsee.kt deleted file mode 100644 index 7dfdf63..0000000 --- a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Griebnitzsee.kt +++ /dev/null @@ -1,47 +0,0 @@ -package de.hpi.cloud.food.crawler.canteens - -import de.hpi.cloud.common.utils.getOrElse -import de.hpi.cloud.food.crawler.CanteenData -import de.hpi.cloud.food.crawler.OpenMensaMeal - -object Griebnitzsee : CanteenData("mensaGriebnitzsee", 62) { - private val LABEL_MAPPING = mapOf( - "Vital" to "vital", - "Vegetarisch" to "vegetarian", - "Vegan" to "vegan", - "Schweinefleisch" to "pork", - "Rindfleisch" to "beef", - "Lamm" to "lamb", - "Knoblauch" to "garlic", - "Gefluegel" to "poultry", - "Fisch" to "fish", - "Alkohol" to "alcohol", - "Outdoor" to null // ignore this label - ) - - override fun findLabels(meal: OpenMensaMeal): List = meal - .notes - .map { it.trim() } - .mapNotNull { note -> - LABEL_MAPPING.getOrElse(note) { - println("Unknown label \"${note}\"") - null - } - } - - fun OpenMensaMeal.categoryMatches(string: String) = category.startsWith(string, ignoreCase = true) - override fun findCounter(meal: OpenMensaMeal): String? = when { - // TODO: translations - meal.categoryMatches("Angebot 1") -> "1" - meal.categoryMatches("Angebot 2") -> "2" - meal.categoryMatches("Angebot 3") -> "3" - meal.categoryMatches("Angebot 4") -> "4" - meal.categoryMatches("Angebot 6") -> "Terrasse" // Hamburger - meal.categoryMatches("Nudeltheke") -> "Nudeltheke" - meal.categoryMatches("Tagessuppe") -> "Suppentheke" - else -> { - println("Unknown meal category/counter \"${meal.category}\" in ${meal.id}") - null - } - } -} diff --git a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Ulf.kt b/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Ulf.kt deleted file mode 100644 index 911718e..0000000 --- a/service-food-crawler/src/main/kotlin/de/hpi/cloud/food/crawler/canteens/Ulf.kt +++ /dev/null @@ -1,17 +0,0 @@ -package de.hpi.cloud.food.crawler.canteens - -import de.hpi.cloud.food.crawler.CanteenData -import de.hpi.cloud.food.crawler.MensaMeal - -object Ulf : CanteenData("ulfsCafe", 112) { - override fun mapReduce(meals: List) = meals - .flatMap { - val alternativeOffer = it.openMensaMeal.notes - .joinToString(separator = " ") - .trim() - if (alternativeOffer.startsWith("oder ", ignoreCase = true)) - listOf(it, it.copy(title = alternativeOffer.substring(5))) - else - listOf(it) - } -} diff --git a/service-food/build.gradle b/service-food/build.gradle index a47aba3..03af3eb 100644 --- a/service-food/build.gradle +++ b/service-food/build.gradle @@ -3,21 +3,33 @@ buildscript { mavenCentral() } dependencies { - classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.3.40") - classpath("com.github.jengelman.gradle.plugins:shadow:5.1.0") + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } - plugins { id "kotlin" id "application" id "com.github.johnrengelman.shadow" version "5.1.0" } +apply plugin: 'kotlinx-serialization' +apply plugin: 'kotlin' + +repositories { + jcenter() + mavenCentral() + maven { url "https://dl.bintray.com/hpi/hpi-cloud-mvn" } +} dependencies { implementation(project(":common")) + + // Crawling + implementation("com.beust:klaxon:5.0.11") + implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" } application { + // mainClassName = "de.hpi.cloud.food.crawler.MainKt" + mainClassName = "de.hpi.cloud.food.FoodServiceKt" } diff --git a/service-food/src/main/kotlin/de/hpi/cloud/food/FoodService.kt b/service-food/src/main/kotlin/de/hpi/cloud/food/FoodService.kt index d1b73df..40dbcce 100644 --- a/service-food/src/main/kotlin/de/hpi/cloud/food/FoodService.kt +++ b/service-food/src/main/kotlin/de/hpi/cloud/food/FoodService.kt @@ -1,149 +1,9 @@ package de.hpi.cloud.food -import com.couchbase.client.java.Bucket -import com.couchbase.client.java.document.json.JsonObject -import com.couchbase.client.java.query.dsl.Expression.s -import com.couchbase.client.java.query.dsl.Expression.x -import com.couchbase.client.java.query.dsl.Sort.asc -import com.couchbase.client.java.query.dsl.functions.DateFunctions.millisToStr -import com.couchbase.client.java.view.ViewQuery -import com.google.protobuf.GeneratedMessageV3 -import de.hpi.cloud.common.Service -import de.hpi.cloud.common.utils.couchbase.* -import de.hpi.cloud.common.utils.getI18nString -import de.hpi.cloud.common.utils.grpc.buildWith -import de.hpi.cloud.common.utils.grpc.buildWithDocument -import de.hpi.cloud.common.utils.grpc.throwException -import de.hpi.cloud.common.utils.grpc.unary -import de.hpi.cloud.common.utils.protobuf.getDateUsingMillis -import de.hpi.cloud.common.utils.protobuf.getMoney -import de.hpi.cloud.common.utils.protobuf.toIsoString -import de.hpi.cloud.food.v1test.* -import io.grpc.Status -import io.grpc.stub.StreamObserver +import de.hpi.cloud.common.Context -fun main(args: Array) { - val service = Service("food", args.firstOrNull()?.toInt()) { FoodServiceImpl(it) } - service.blockUntilShutdown() -} - -class FoodServiceImpl(private val bucket: Bucket) : FoodServiceGrpc.FoodServiceImplBase() { - companion object { - const val DESIGN_RESTAURANT = "restaurant" - const val DESIGN_MENU_ITEM = "menuItem" - const val DESIGN_LABEL = "label" - } - - // region Restaurant - override fun listRestaurants( - request: ListRestaurantsRequest?, - responseObserver: StreamObserver? - ) = unary(request, responseObserver, "listRestaurants") { req -> - val (restaurants, newToken) = ViewQuery.from(DESIGN_RESTAURANT, VIEW_BY_ID) - .paginate(bucket, req.pageSize, req.pageToken) { it.parseRestaurant(req) } - - ListRestaurantsResponse.newBuilder().buildWith { - addAllRestaurants(restaurants) - nextPageToken = newToken - } - } - - override fun getRestaurant(request: GetRestaurantRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getRestaurant") { req -> - if (req.id.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("Restaurant ID is required") - - bucket.get(DESIGN_RESTAURANT, VIEW_BY_ID, req.id) - ?.document()?.content()?.parseRestaurant(req) - ?: Status.NOT_FOUND.throwException("Restaurant with ID ${req.id} not found") - } - - private fun JsonObject.parseRestaurant(request: GeneratedMessageV3) = - Restaurant.newBuilder().buildWithDocument(this) { - id = getString(KEY_ID) - title = it.getI18nString("title", request) - } - // endregion - - // region MenuItem - override fun listMenuItems( - request: ListMenuItemsRequest?, - responseObserver: StreamObserver? - ) = unary(request, responseObserver, "listMenuItems") { req -> - val restaurantId = req.restaurantId?.trim()?.takeIf { it.isNotEmpty() } - val date = if (req.hasDate()) req.date else null - - val (menuItems, newToken) = paginate(bucket, { - where( - and( - x(KEY_TYPE).eq(s("menuItem")), - restaurantId?.let { v("restaurantId").eq(s(restaurantId)) }, - date?.let { - millisToStr(v("date", "millis"), "1111-11-11") - .eq(s(it.toIsoString())) - } - ) - ) - .orderBy( - *descTimestamp(v("date")), - asc(v("offerName")) - ) - }, req.pageSize, req.pageToken) { it.parseMenuItem(req) } - - ListMenuItemsResponse.newBuilder().buildWith { - addAllItems(menuItems) - nextPageToken = newToken - } - } - - override fun getMenuItem(request: GetMenuItemRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "getMenuItem") { req -> - if (req.id.isNullOrEmpty()) Status.INVALID_ARGUMENT.throwException("Argument ID is required") - - bucket.get(DESIGN_MENU_ITEM, VIEW_BY_ID, req.id) - ?.document()?.content()?.parseMenuItem(req) - ?: Status.NOT_FOUND.throwException("MenuItem with ID ${req.id} not found") - } - - private fun JsonObject.parseMenuItem(request: GeneratedMessageV3) = - MenuItem.newBuilder().buildWithDocument(this) { - id = getString(KEY_ID) - restaurantId = it.getString("restaurantId") - it.getDateUsingMillis("date")?.let { d -> date = d } - it.getI18nString("counter", request)?.let { c -> counter = c } - it.getObject("prices").let { prices -> - putAllPrices(prices.names.map { p -> p to prices.getMoney(p) }.toMap()) - } - title = it.getI18nString("title", request) - addAllLabelIds(it.getStringArray("labelIds").filterNotNull()) - } - // endregion - - // region Label - override fun listLabels(request: ListLabelsRequest?, responseObserver: StreamObserver?) = - unary(request, responseObserver, "listLabels") { req -> - val (labels, newToken) = ViewQuery.from(DESIGN_LABEL, VIEW_BY_ID) - .paginate(bucket, req.pageSize, req.pageToken) { it.parseLabel(req) } - - ListLabelsResponse.newBuilder().buildWith { - addAllLabels(labels) - nextPageToken = newToken - } - } - - override fun getLabel(request: GetLabelRequest?, responseObserver: StreamObserver