Skip to content

Commit

Permalink
Put accuracy report code into separate source set (#68)
Browse files Browse the repository at this point in the history
  • Loading branch information
pemistahl committed Aug 11, 2020
1 parent c408f74 commit fd7076e
Show file tree
Hide file tree
Showing 561 changed files with 40 additions and 13 deletions.
31 changes: 21 additions & 10 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,26 @@ sourceSets {
exclude("training-data/**")
}
}
create("accuracyReport") {
compileClasspath += sourceSets.main.get().output
runtimeClasspath += sourceSets.main.get().output
}
}

val accuracyReportImplementation by configurations.getting {
extendsFrom(configurations.testImplementation.get())
}

configurations["accuracyReportRuntimeOnly"].extendsFrom(configurations.runtimeOnly.get())

compileTestKotlin.kotlinOptions.jvmTarget = "1.8"

tasks.withType<Test> {
useJUnitPlatform { failFast = true }
tasks.named("compileAccuracyReportKotlin", KotlinCompile::class) {
kotlinOptions.jvmTarget = "1.8"
}

tasks.test {
filter { includeTestsMatching("*Test") }
tasks.withType<Test> {
useJUnitPlatform { failFast = true }
}

tasks.jacocoTestReport {
Expand All @@ -94,9 +104,11 @@ tasks.jacocoTestReport {
}
}

tasks.register<Test>("writeAccuracyReports") {
tasks.register<Test>("accuracyReport") {
group = linguaTaskGroup
description = "Runs Lingua on provided test data, and writes detection accuracy reports for each language."
testClassesDirs = sourceSets["accuracyReport"].output.classesDirs
classpath = sourceSets["accuracyReport"].runtimeClasspath

val allowedDetectors = linguaSupportedDetectors.split(',')
val detectors = if (project.hasProperty("detectors"))
Expand Down Expand Up @@ -297,11 +309,10 @@ dependencies {
testImplementation("org.assertj:assertj-core:3.16.1")
testImplementation("io.mockk:mockk:1.10.0")

testImplementation("com.optimaize.languagedetector:language-detector:0.6")
testImplementation("org.apache.opennlp:opennlp-tools:1.9.3")
testImplementation("org.apache.tika:tika-langdetect:1.24.1")

testImplementation("org.slf4j:slf4j-nop:1.7.30")
accuracyReportImplementation("com.optimaize.languagedetector:language-detector:0.6")
accuracyReportImplementation("org.apache.opennlp:opennlp-tools:1.9.3")
accuracyReportImplementation("org.apache.tika:tika-langdetect:1.24.1")
accuracyReportImplementation("org.slf4j:slf4j-nop:1.7.30")
}

python {
Expand Down
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ import com.github.pemistahl.lingua.api.Language
import com.github.pemistahl.lingua.api.Language.CHINESE
import com.github.pemistahl.lingua.api.Language.UNKNOWN
import com.github.pemistahl.lingua.api.LanguageDetectorBuilder
import com.github.pemistahl.lingua.internal.util.extension.incrementCounter
import com.github.pemistahl.lingua.report.LanguageDetectorImplementation.LINGUA
import com.github.pemistahl.lingua.report.LanguageDetectorImplementation.OPENNLP
import com.github.pemistahl.lingua.report.LanguageDetectorImplementation.OPTIMAIZE
Expand Down Expand Up @@ -192,7 +191,7 @@ abstract class AbstractLanguageDetectionAccuracyReport(
Language.getByIsoCode639_3(isoCode)
}
}
statistics.incrementCounter(detectedLanguage)
statistics[detectedLanguage] = statistics.getOrDefault(detectedLanguage, 0) + 1
}

private fun computeAccuracy(languageCount: Int, totalLanguagesCount: Int) =
Expand Down
Loading

0 comments on commit fd7076e

Please sign in to comment.