diff --git a/.github/workflows/example-sanity-check.yml b/.github/workflows/example-sanity-check.yml index cb727e6995..12798e29f5 100644 --- a/.github/workflows/example-sanity-check.yml +++ b/.github/workflows/example-sanity-check.yml @@ -25,39 +25,28 @@ jobs: with: access_token: ${{ github.token }} - uses: actions/checkout@v2 - - uses: ./.github/actions/update-deps-linux - - uses: ./.github/actions/install-protobuf-linux - - uses: ./.github/actions/install-cmake-linux - name: Set up Java for publishing to GitHub Packages uses: actions/setup-java@v1 with: java-version: 1.8 - server-id: sonatype-nexus-snapshots - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build on linux-arm64 shell: bash env: DEBIAN_FRONTEND: noninteractive GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DEPLOY: 1 - BUILD_USING_MAVEN: 1 - TARGET_OS: linux - CURRENT_TARGET: arm64 - PUBLISH_TO: ossrh - DEPLOY_TO: ossrh - MAVEN_USERNAME: ${{ secrets.SONATYPE_USER_1 }} - MAVEN_PASSWORD: ${{ secrets.SONATYPE_USER1_PASS }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} - run: | mvn --version - cmake --version - protoc --version for folder in `ls`; do - mvn clean test + if [[ -d "$folder" ]]; then + cd "$folder" + if test -f "pom.xml"; then + mvn clean test + fi + cd .. + fi + + + done diff --git a/.github/workflows/run-sanity-checks.sh b/.github/workflows/run-sanity-checks.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/README.md b/README.md index 1714ae79fa..43e14447fc 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ We do not monitor the github issues of this repository very often. The **Eclipse Deeplearning4J** (DL4J) ecosystem is a set of projects intended to support all the needs of a JVM based deep learning application. This means starting with the raw data, loading and preprocessing it from wherever and whatever format it is in to building and tuning a wide variety of simple and complex deep learning networks. The DL4J stack comprises of: -- **DL4J**: High level API to build MultiLayerNetworks and ComputationGraphs with a variety of layers, including custom ones. Supports importing Keras models from h5, including tf.keras models (as of 1.0.0-beta7) and also supports distributed training on Apache Spark +- **DL4J**: High level API to build MultiLayerNetworks and ComputationGraphs with a variety of layers, including custom ones. Supports importing Keras models from h5, including tf.keras models (as of 1.0.0-M1.1) and also supports distributed training on Apache Spark - **ND4J**: General purpose linear algebra library with over 500 mathematical, linear algebra and deep learning operations. ND4J is based on the highly-optimized C++ codebase LibND4J that provides CPU (AVX2/512) and GPU (CUDA) support and acceleration by libraries such as OpenBLAS, OneDNN (MKL-DNN), cuDNN, cuBLAS, etc - **SameDiff** : Part of the ND4J library, SameDiff is our automatic differentiation / deep learning framework. SameDiff uses a graph-based (define then run) approach, similar to TensorFlow graph mode. Eager graph (TensorFlow 2.x eager/PyTorch) graph execution is planned. SameDiff supports importing TensorFlow frozen model format .pb (protobuf) models. Import for ONNX, TensorFlow SavedModel and Keras models are planned. Deeplearning4j also has full SameDiff support for easily writing custom layers and loss functions. - **DataVec**: ETL for machine learning data in a wide variety of formats and files (HDFS, Spark, Images, Video, Audio, CSV, Excel etc) @@ -40,6 +40,17 @@ Users can also refer to the [simple sample project provided](./mvn-project-templ Build tools are considered standard software engineering best practice. Besides this the complexities posed by the projects in the DL4J ecosystem make dependencies too difficult to manage manually. All the projects in the DL4J ecosystem can be used with other build tools like Gradle, SBT etc. More information on that can be found [here](https://deeplearning4j.konduit.ai/config/buildtools). +## Support + +For help with the examples, please go to our [support forum](https://community.konduit.ai/) + +Note for users of 1.0.0-beta7 and prior, some examples and modules have been removed to reflect +changes in the framework's direction. Please see and comment on our post [here](https://community.konduit.ai/t/upcoming-removal-of-modules-and-roadmap-changes/1240) + +If you would like a workaround for something you may be missing, +please feel free to post on the forums, and we will do what we can to help you. + + ## Example Content Projects are based on what functionality the included examples demonstrate to the user and not necessarily which library in the DL4J stack the functionality lives in. @@ -69,9 +80,6 @@ This project contains a set of examples that demonstrate how raw data in various - [nd4j-ndarray-examples](nd4j-ndarray-examples/README.md) This project contains a set of examples that demonstrate how to manipulate NDArrays. The functionality of ND4J demonstrated here can be likened to NumPy. -- [arbiter-examples](arbiter-examples/README.md) -This project contains a set of examples that demonstrate usage of the Arbiter library for hyperparameter tuning of Deeplearning4J neural networks. - - [rl4j-examples](rl4j-examples/README.md) This project contains examples of using RL4J, the reinforcement learning library in DL4J. diff --git a/android-examples/README.md b/android-examples/README.md index 4163b9262d..fe4efcbe82 100644 --- a/android-examples/README.md +++ b/android-examples/README.md @@ -8,14 +8,6 @@ The build configuration for Android can be found in [app/build.gradle](./app/bui [Go back](../README.md) to the main repository page to explore other features/functionality of the **Eclipse Deeplearning4J** ecosystem. File an issue [here](https://github.com/eclipse/deeplearning4j-examples/issues) to request new features. -# Known Issues -Due to an unfortunately timed change, the 1.0.0-beta7 release doesn't work well with most Android devices. For this reason, this example project uses the SNAPSHOT version. - -Again unfortunately, gradle has issues retrieving SNAPSHOT versions when combined with classifiers, which are used here to retrieve only the android specific backend dependencies. - -If you want to run this example locally, you will therefore need to have Maven installed in order to download those specific dependencies. - -Once you have installed Maven, you will need to run the following commands from the command line to download the correct android backend files: ``` mvn dependency:get -DremoteRepositories=snapshots::::https://oss.sonatype.org/content/repositories/snapshots -Dartifact=org.nd4j:nd4j-native:1.0.0-SNAPSHOT:jar:android-arm diff --git a/android-examples/app/build.gradle b/android-examples/app/build.gradle index b5ad7c806d..280f04581d 100644 --- a/android-examples/app/build.gradle +++ b/android-examples/app/build.gradle @@ -11,7 +11,7 @@ android { versionCode 1 versionName "1.0" - testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -27,10 +27,10 @@ android { } dependencies { - def dl4jVersion = '1.0.0-beta7' - def openblasVersion = '0.3.9-1.5.3' - def opencvVersion = '4.3.0-1.5.3' - def leptonicaVersion = '1.79.0-1.5.3' + def dl4jVersion = '1.0.0-M1.1' + def openblasVersion = '0.3.10-1.5.5' + def opencvVersion = '4.5.1-1.5.5' + def leptonicaVersion = '1.80.0-1.5.5' implementation fileTree(dir: 'libs', include: ['*.jar']) @@ -45,13 +45,14 @@ dependencies { exclude group: 'org.bytedeco', module: 'leptonica-platform' exclude group: 'org.bytedeco', module: 'hdf5-platform' exclude group: 'org.nd4j', module: 'nd4j-base64' + exclude group: 'org.nd4j', module: 'nd4j-api' } implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion - implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-arm" - implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-arm64" - implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-x86" - implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-x86_64" + implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-arm" + implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-arm64" + implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-x86" + implementation group: 'org.nd4j', name: 'nd4j-native', version: dl4jVersion, classifier: "android-x86_64" implementation group: 'org.bytedeco', name: 'openblas', version: openblasVersion implementation group: 'org.bytedeco', name: 'openblas', version: openblasVersion, classifier: "android-arm" implementation group: 'org.bytedeco', name: 'openblas', version: openblasVersion, classifier: "android-arm64" @@ -69,4 +70,4 @@ dependencies { implementation group: 'org.bytedeco', name: 'leptonica', version: leptonicaVersion, classifier: "android-x86_64" annotationProcessor group: 'org.projectlombok', name: 'lombok', version: '1.18.4' -} \ No newline at end of file +} diff --git a/android-examples/app/src/androidTest/java/com/example/androidDl4jClassifier/ExampleInstrumentedTest.java b/android-examples/app/src/androidTest/java/com/example/androidclassifier/ExampleInstrumentedTest.java similarity index 94% rename from android-examples/app/src/androidTest/java/com/example/androidDl4jClassifier/ExampleInstrumentedTest.java rename to android-examples/app/src/androidTest/java/com/example/androidclassifier/ExampleInstrumentedTest.java index 49d122bea4..10e124502a 100644 --- a/android-examples/app/src/androidTest/java/com/example/androidDl4jClassifier/ExampleInstrumentedTest.java +++ b/android-examples/app/src/androidTest/java/com/example/androidclassifier/ExampleInstrumentedTest.java @@ -1,4 +1,4 @@ -package com.example.androidDl4jClassifier; +package com.example.androidclassifier; import android.content.Context; diff --git a/android-examples/app/src/main/AndroidManifest.xml b/android-examples/app/src/main/AndroidManifest.xml index 6099bbe213..1a9f016e76 100644 --- a/android-examples/app/src/main/AndroidManifest.xml +++ b/android-examples/app/src/main/AndroidManifest.xml @@ -1,6 +1,6 @@ + package="com.example.androidclassifier"> - + @@ -19,4 +19,4 @@ - \ No newline at end of file + diff --git a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/MainActivity.java b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/MainActivity.java similarity index 94% rename from android-examples/app/src/main/java/com/example/androidDl4jClassifier/MainActivity.java rename to android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/MainActivity.java index b16ea2773d..1e98690ad4 100644 --- a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/MainActivity.java +++ b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/MainActivity.java @@ -1,10 +1,11 @@ -package com.example.androidDl4jClassifier; +package org.deeplearning4j.examples.androidclassifier; import androidx.appcompat.app.AppCompatActivity; import android.os.Bundle; import android.view.Menu; import android.view.MenuItem; +import com.example.androidclassifier.R; public class MainActivity extends AppCompatActivity { diff --git a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/ScatterView.java b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/ScatterView.java similarity index 98% rename from android-examples/app/src/main/java/com/example/androidDl4jClassifier/ScatterView.java rename to android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/ScatterView.java index 2c3fd54202..0a29b9cc10 100644 --- a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/ScatterView.java +++ b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/ScatterView.java @@ -1,4 +1,4 @@ -package com.example.androidDl4jClassifier; +package org.deeplearning4j.examples.androidclassifier; import android.content.Context; import android.graphics.Canvas; diff --git a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/TrainingTask.java b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/TrainingTask.java similarity index 99% rename from android-examples/app/src/main/java/com/example/androidDl4jClassifier/TrainingTask.java rename to android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/TrainingTask.java index f2e569da74..d2f4cff26b 100644 --- a/android-examples/app/src/main/java/com/example/androidDl4jClassifier/TrainingTask.java +++ b/android-examples/app/src/main/java/org/deeplearning4j/examples/androidclassifier/TrainingTask.java @@ -1,4 +1,4 @@ -package com.example.androidDl4jClassifier; +package org.deeplearning4j.examples.androidclassifier; import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; diff --git a/android-examples/app/src/main/res/layout/activity_main.xml b/android-examples/app/src/main/res/layout/activity_main.xml index 4a97e49cc6..3738b8604e 100644 --- a/android-examples/app/src/main/res/layout/activity_main.xml +++ b/android-examples/app/src/main/res/layout/activity_main.xml @@ -4,16 +4,16 @@ xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" - tools:context=".MainActivity"> + tools:context="org.deeplearning4j.examples.androidclassifier.MainActivity"> - \ No newline at end of file + diff --git a/android-examples/app/src/test/java/com/example/androidDl4jClassifier/ExampleUnitTest.java b/android-examples/app/src/test/java/com/example/androidDl4jClassifier/ExampleUnitTest.java deleted file mode 100644 index 71095c0b44..0000000000 --- a/android-examples/app/src/test/java/com/example/androidDl4jClassifier/ExampleUnitTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.example.androidDl4jClassifier; - -import org.junit.Test; - -import static org.junit.Assert.*; - -/** - * Example local unit test, which will execute on the development machine (host). - * - * @see Testing documentation - */ -public class ExampleUnitTest { - @Test - public void addition_isCorrect() { - assertEquals(4, 2 + 2); - } -} \ No newline at end of file diff --git a/cuda-specific-examples/README.md b/cuda-specific-examples/README.md deleted file mode 100644 index 62dde4afee..0000000000 --- a/cuda-specific-examples/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## Eclipse Deeplearning4j: CUDA Specific Examples - -Switching from a CPU only backend to a GPU backend is as simple as changing one dependency - one line in the pom.xml file for Maven users. Instead of specifying the nd4j-native-platform module specify the nd4j-cuda-X-platform where X indicated the version of CUDA. It is recommended to install cuDNN for better GPU performance. Runs will log warnings if cuDNN is not found. For more information, please refer to documentation [here](https://deeplearning4j.konduit.ai/config/backends/config-cudnn#using-deeplearning-4-j-with-cudnn) - -Users with access to systems with multiple gpus can use DL4J to further speed up the training process by training the models in parallel on them. Ideally these GPUs have the same speed and networking capabilities. This project contains a set of examples that demonstrate how to leverage performance from a multiple gpus setup. More documentation can be found [here](https://deeplearning4j.konduit.ai/getting-started/tutorials/using-multiple-gpus) - -[Go back](../README.md) to the main repository page to explore other features/functionality of the **Eclipse Deeplearning4J** ecosystem. File an issue [here](https://github.com/eclipse/deeplearning4j-examples/issues) to request new features. - -The list of examples in this project and what they demonstrate are briefly described below. This is also the recommended order to explore them in. - -## QUICKSTART -* [MultiGPULeNetMNIST.java](./src/main/java/org/deeplearning4j/examples/multigpu/quickstart/MultiGPULeNetMNIST.java) -Introduction to ParallelWrapper by modifying the original [LenetMnistExample](./../dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/convolution/LeNetMNIST.java) -* [GradientsSharingLeNetMNIST.java](./src/main/java/org/deeplearning4j/examples/multigpu/quickstart/GradientsSharingLeNetMNIST.java) -Uses gradient sharing instead of the default averaging every n iterations. More information on the gradient sharing algorithm can be found [here](https://deeplearning4j.konduit.ai/distributed-deep-learning/intro) -* [GradientSharingVGG16TinyImageNet.java](./src/main/java/org/deeplearning4j/examples/multigpu/quickstart/GradientSharingVGG16TinyImageNet.java) -Gradient sharing with VGG16 on TinyImageNet - -## ADVANCED -* [ImdbReviewClassificationRNN.java](./src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/ImdbReviewClassificationRNN.java) -A multiple gpus version of the example of the same name in the dl4j-examples repo [here](./../dl4j-examples//src/main/java/org/deeplearning4j/examples/advanced/modelling/textclassification/pretrainedword2vec/ImdbReviewClassificationRNN.java) This example also includes how to presave the dataset to save time when training on multiple epochs. -* [GenerateTxtModel.java](./src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/GenerateTxtModel.java) -CharModelling: A multiple gpus version of the example of the same name in the dl4j-examples repo, [here](./../dl4j-examples/src/main/java/org/deeplearning4j/examples/advanced/modelling/charmodelling/generatetext/GenerateTxtModel.java). -* [FeaturizedPreSave.java](./src/main/java/org/deeplearning4j/examples/multigpu/advanced/transferlearning/vgg16/FeaturizedPreSave.java) & [FitFromFeaturized.java](./src/main/java/org/deeplearning4j/examples/multigpu/advanced/transferlearning/vgg16/FitFromFeaturized.java) -Transferlearning: A multiple gpus version of the example of the same name in the dl4j-examples repo [here](./../dl4j-examples/src/main/java/org/deeplearning4j/examples/advanced/features/transferlearning/editlastlayer/presave) - diff --git a/cuda-specific-examples/pom.xml b/cuda-specific-examples/pom.xml deleted file mode 100644 index dfa4459bbb..0000000000 --- a/cuda-specific-examples/pom.xml +++ /dev/null @@ -1,173 +0,0 @@ - - - - - 4.0.0 - org.deeplearning4j - dl4j-cuda-specific-examples - 1.0.0-beta7 - DeepLearning4j CUDA special examples - - 1.0.0-beta7 - nd4j-cuda-10.2 - 1.8 - 1.4.0 - 2.4.3 - 1.0.23 - 1.1.7 - UTF-8 - - - - - - org.nd4j - nd4j-cuda-10.2 - ${dl4j-master.version} - - - org.nd4j - nd4j-cuda-10.2 - ${dl4j-master.version} - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.42.Final - - - - - - - - org.deeplearning4j - deeplearning4j-parallel-wrapper - ${dl4j-master.version} - - - - org.deeplearning4j - deeplearning4j-core - ${dl4j-master.version} - - - org.deeplearning4j - deeplearning4j-ui - ${dl4j-master.version} - - - org.deeplearning4j - deeplearning4j-zoo - ${dl4j-master.version} - - - org.nd4j - ${nd4j.backend} - - - - datavec-data-codec - org.datavec - ${dl4j-master.version} - - - - ch.qos.logback - logback-classic - ${logback.version} - - - - - - - org.codehaus.mojo - exec-maven-plugin - ${exec-maven-plugin.version} - - - - exec - - - - - java - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} - - true - ${shadedClassifier} - true - - - *:* - - org/datanucleus/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - package - - shade - - - - - reference.conf - - - - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - ${java.version} - ${java.version} - - - - - - diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/CharacterIterator.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/CharacterIterator.java deleted file mode 100644 index eaf55f8fd0..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/CharacterIterator.java +++ /dev/null @@ -1,251 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.charmodelling; - -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.DataSetPreProcessor; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.util.*; - -/** A simple DataSetIterator for use in the LSTMCharModellingExample. - * Given a text file and a few options, generate feature vectors and labels for training, - * where we want to predict the next character in the sequence.
- * This is done by randomly choosing a position in the text file, at offsets of 0, exampleLength, 2*exampleLength, etc - * to start each sequence. Then we convert each character to an index, i.e., a one-hot vector. - * Then the character 'a' becomes [1,0,0,0,...], 'b' becomes [0,1,0,0,...], etc - * - * Feature vectors and labels are both one-hot vectors of same length - * @author Alex Black - */ -public class CharacterIterator implements DataSetIterator { - //Valid characters - private char[] validCharacters; - //Maps each character to an index ind the input/output - private Map charToIdxMap; - //All characters of the input file (after filtering to only those that are valid - private char[] fileCharacters; - //Length of each example/minibatch (number of characters) - private int exampleLength; - //Size of each minibatch (number of examples) - private int miniBatchSize; - private Random rng; - //Offsets for the start of each example - private LinkedList exampleStartOffsets = new LinkedList<>(); - - /** - * @param textFilePath Path to text file to use for generating samples - * @param textFileEncoding Encoding of the text file. Can try Charset.defaultCharset() - * @param miniBatchSize Number of examples per mini-batch - * @param exampleLength Number of characters in each input/output vector - * @param validCharacters Character array of valid characters. Characters not present in this array will be removed - * @param rng Random number generator, for repeatability if required - * @throws IOException If text file cannot be loaded - */ - public CharacterIterator(String textFilePath, Charset textFileEncoding, int miniBatchSize, int exampleLength, - char[] validCharacters, Random rng) throws IOException { - if( !new File(textFilePath).exists()) throw new IOException("Could not access file (does not exist): " + textFilePath); - if( miniBatchSize <= 0 ) throw new IllegalArgumentException("Invalid miniBatchSize (must be >0)"); - this.validCharacters = validCharacters; - this.exampleLength = exampleLength; - this.miniBatchSize = miniBatchSize; - this.rng = rng; - - //Store valid characters is a map for later use in vectorization - charToIdxMap = new HashMap<>(); - for( int i=0; i lines = Files.readAllLines(new File(textFilePath).toPath(),textFileEncoding); - int maxSize = lines.size(); //add lines.size() to account for newline characters at end of each line - for( String s : lines ) maxSize += s.length(); - char[] characters = new char[maxSize]; - int currIdx = 0; - for( String s : lines ){ - char[] thisLine = s.toCharArray(); - for (char aThisLine : thisLine) { - if (!charToIdxMap.containsKey(aThisLine)) continue; - characters[currIdx++] = aThisLine; - } - if(newLineValid) characters[currIdx++] = '\n'; - } - - if( currIdx == characters.length ){ - fileCharacters = characters; - } else { - fileCharacters = Arrays.copyOfRange(characters, 0, currIdx); - } - if( exampleLength >= fileCharacters.length ) throw new IllegalArgumentException("exampleLength="+exampleLength - +" cannot exceed number of valid characters in file ("+fileCharacters.length+")"); - - int nRemoved = maxSize - fileCharacters.length; - System.out.println("Loaded and converted file: " + fileCharacters.length + " valid characters of " - + maxSize + " total characters (" + nRemoved + " removed)"); - - initializeOffsets(); - } - - /** A minimal character set, with a-z, A-Z, 0-9 and common punctuation etc */ - public static char[] getMinimalCharacterSet(){ - List validChars = new LinkedList<>(); - for(char c='a'; c<='z'; c++) validChars.add(c); - for(char c='A'; c<='Z'; c++) validChars.add(c); - for(char c='0'; c<='9'; c++) validChars.add(c); - char[] temp = {'!', '&', '(', ')', '?', '-', '\'', '"', ',', '.', ':', ';', ' ', '\n', '\t'}; - for( char c : temp ) validChars.add(c); - char[] out = new char[validChars.size()]; - int i=0; - for( Character c : validChars ) out[i++] = c; - return out; - } - - /** As per getMinimalCharacterSet(), but with a few extra characters */ - public static char[] getDefaultCharacterSet(){ - List validChars = new LinkedList<>(); - for(char c : getMinimalCharacterSet() ) validChars.add(c); - char[] additionalChars = {'@', '#', '$', '%', '^', '*', '{', '}', '[', ']', '/', '+', '_', - '\\', '|', '<', '>'}; - for( char c : additionalChars ) validChars.add(c); - char[] out = new char[validChars.size()]; - int i=0; - for( Character c : validChars ) out[i++] = c; - return out; - } - - public char convertIndexToCharacter( int idx ){ - return validCharacters[idx]; - } - - public int convertCharacterToIndex( char c ){ - return charToIdxMap.get(c); - } - - public char getRandomCharacter(){ - return validCharacters[(int) (rng.nextDouble()*validCharacters.length)]; - } - - public boolean hasNext() { - return exampleStartOffsets.size() > 0; - } - - public DataSet next() { - return next(miniBatchSize); - } - - public DataSet next(int num) { - if( exampleStartOffsets.size() == 0 ) throw new NoSuchElementException(); - - int currMinibatchSize = Math.min(num, exampleStartOffsets.size()); - //Allocate space: - //Note the order here: - // dimension 0 = number of examples in minibatch - // dimension 1 = size of each vector (i.e., number of characters) - // dimension 2 = length of each time series/example - INDArray input = Nd4j.create(new int[]{currMinibatchSize,validCharacters.length,exampleLength}, 'f'); - INDArray labels = Nd4j.create(new int[]{currMinibatchSize,validCharacters.length,exampleLength}, 'f'); - - for( int i=0; i getLabels() { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - -} diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/GenerateTxtModel.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/GenerateTxtModel.java deleted file mode 100644 index ec980cee33..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/charmodelling/GenerateTxtModel.java +++ /dev/null @@ -1,247 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.charmodelling; - -import org.apache.commons.io.FileUtils; -import org.deeplearning4j.nn.api.Layer; -import org.deeplearning4j.nn.api.Model; -import org.deeplearning4j.nn.conf.BackpropType; -import org.deeplearning4j.nn.conf.ComputationGraphConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.LSTM; -import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; -import org.deeplearning4j.nn.graph.ComputationGraph; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.api.IterationListener; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.deeplearning4j.parallelism.ParallelWrapper; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.BooleanIndexing; -import org.nd4j.linalg.indexing.conditions.Conditions; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.nio.charset.Charset; -import java.util.Random; - -//import org.nd4j.jita.conf.CudaEnvironment; - -/**LSTM Character modelling example - * @author Alex Black - - Example: Train a LSTM RNN to generates text, one character at a time. - This example is somewhat inspired by Andrej Karpathy's blog post, - "The Unreasonable Effectiveness of Recurrent Neural Networks" - http://karpathy.github.io/2015/05/21/rnn-effectiveness/ - - This example is set up to train on the Complete Works of William Shakespeare, downloaded - from Project Gutenberg. Training on other text sources should be relatively easy to implement. - - For more details on RNNs in DL4J, see the following: - https://deeplearning4j.konduit.ai/models/recurrent - */ -public class GenerateTxtModel { - public static void main( String[] args ) throws Exception { - int seed = 12345; - int lstmLayerSize = 200; //Number of units in each LSTM layer - int miniBatchSize = 32; //Size of mini batch to use when training - int exampleLength = 1000; //Length of each training example sequence to use. This could certainly be increased - int tbpttLength = 50; //Length for truncated backpropagation through time. i.e., do parameter updates ever 50 characters - int nSamplesToGenerate = 4; //Number of samples to generate after each training epoch - int nCharactersToSample = 300; //Length of each sample to generate - String generationInitialization = null; //Optional character initialization; a random character is used if null - - // Above is Used to 'prime' the LSTM with a character sequence to continue/complete. - // Initialization characters must all be in CharacterIterator.getMinimalCharacterSet() by default - Random rng = new Random(seed); - - //Get a DataSetIterator that handles vectorization of text into something we can use to train - // our LSTM network. - CharacterIterator iter = getShakespeareIterator(miniBatchSize,exampleLength); - int nOut = iter.totalOutcomes(); - - //Set up network configuration: - ComputationGraphConfiguration conf = new NeuralNetConfiguration.Builder() - .seed(12345) - .l2(0.0001) - .weightInit(WeightInit.XAVIER) - .updater(new Adam(0.005)) - .graphBuilder() - .addInputs("input") //Give the input a name. For a ComputationGraph with multiple inputs, this also defines the input array orders - //First layer: name "first", with inputs from the input called "input" - .addLayer("first", new LSTM.Builder().nIn(iter.inputColumns()).nOut(lstmLayerSize) - .activation(Activation.TANH).build(),"input") - //Second layer, name "second", with inputs from the layer called "first" - .addLayer("second", new LSTM.Builder().nIn(lstmLayerSize).nOut(lstmLayerSize) - .activation(Activation.TANH).build(),"first") - //Output layer, name "outputlayer" with inputs from the two layers called "first" and "second" - .addLayer("outputLayer", new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX) - .nIn(2*lstmLayerSize).nOut(nOut).build(),"first","second") - .setOutputs("outputLayer") //List the output. For a ComputationGraph with multiple outputs, this also defines the input array orders - .backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(tbpttLength).tBPTTBackwardLength(tbpttLength) - .build(); - - ComputationGraph net = new ComputationGraph(conf); - net.init(); - net.setListeners(new ScoreIterationListener(1), new IterationListener() { - @Override - public void iterationDone(Model model, int iteration, int epoch) { - if (iteration % 20 == 0) { - System.out.println("--------------------"); - System.out.println("Sampling characters from network given initialization \"" + (generationInitialization == null ? "" : generationInitialization) + "\""); - String[] samples = sampleCharactersFromNetwork(generationInitialization, (ComputationGraph) model, iter, rng, nCharactersToSample, nSamplesToGenerate); - for (int j = 0; j < samples.length; j++) { - System.out.println("----- Sample " + j + " -----"); - System.out.println(samples[j]); - System.out.println(); - } - } - } - }); - - - //Print the number of parameters in the network (and for each layer) - Layer[] layers = net.getLayers(); - long totalNumParams = 0; - for( int i= 0; i < layers.length; i++) { - long nParams = layers[i].numParams(); - System.out.println("Number of parameters in layer " + i + ": " + nParams); - totalNumParams += nParams; - } - - System.out.println("Total number of network parameters: " + totalNumParams); - - - // ParallelWrapper will take care of load balancing between GPUs. - ParallelWrapper wrapper = new ParallelWrapper.Builder(net) - // DataSets prefetching options. Set this value with respect to number of actual devices - .prefetchBuffer(24) - - // set number of workers equal to number of available devices. x1-x2 are good values to start with - .workers(2) - - .build(); - - wrapper.fit(iter); - - - System.out.println("\n\nExample complete"); - } - - /** Downloads Shakespeare training data and stores it locally (temp directory). Then set up and return a simple - * DataSetIterator that does vectorization based on the text. - * @param miniBatchSize Number of text segments in each training mini-batch - * @param sequenceLength Number of characters in each text segment. - */ - public static CharacterIterator getShakespeareIterator(int miniBatchSize, int sequenceLength) throws Exception { - //The Complete Works of William Shakespeare - //5.3MB file in UTF-8 Encoding, ~5.4 million characters - //https://www.gutenberg.org/ebooks/100 - String url = "https://s3.amazonaws.com/dl4j-distribution/pg100.txt"; - String tempDir = System.getProperty("java.io.tmpdir"); - String fileLocation = tempDir + "/Shakespeare.txt"; //Storage location from downloaded file - File f = new File(fileLocation); - if( !f.exists() ){ - FileUtils.copyURLToFile(new URL(url), f); - System.out.println("File downloaded to " + f.getAbsolutePath()); - } else { - System.out.println("Using existing text file at " + f.getAbsolutePath()); - } - - if(!f.exists()) throw new IOException("File does not exist: " + fileLocation); //Download problem? - - char[] validCharacters = CharacterIterator.getMinimalCharacterSet(); //Which characters are allowed? Others will be removed - return new CharacterIterator(fileLocation, Charset.forName("UTF-8"), - miniBatchSize, sequenceLength, validCharacters, new Random(12345)); - } - - /** Generate a sample from the network, given an (optional, possibly null) initialization. Initialization - * can be used to 'prime' the RNN with a sequence you want to extend/continue.
- * Note that the initalization is used for all samples - * @param initialization String, may be null. If null, select a random character as initialization for all samples - * @param charactersToSample Number of characters to sample from network (excluding initialization) - * @param net MultiLayerNetwork with one or more LSTM/RNN layers and a softmax output layer - * @param iter CharacterIterator. Used for going from indexes back to characters - */ - private static String[] sampleCharactersFromNetwork(String initialization, ComputationGraph net, - CharacterIterator iter, Random rng, int charactersToSample, int numSamples ){ - //Set up initialization. If no initialization: use a random character - if( initialization == null ){ - initialization = String.valueOf(iter.getRandomCharacter()); - } - - //Create input for initialization - INDArray initializationInput = Nd4j.zeros(numSamples, iter.inputColumns(), initialization.length()); - char[] init = initialization.toCharArray(); - for( int i=0; i= 100) { - throw new IllegalArgumentException("Percentage of data set aside for training has to be less than 100%. Test percentage = 100 - training percentage, has to be greater than 0"); - } - InputSplit[] filesInDirSplit = filesInDir.sample(pathFilter, trainPerc, 100-trainPerc); - trainData = filesInDirSplit[0]; - testData = filesInDirSplit[1]; - } - - private static DataSetIterator makeIterator(InputSplit split) throws IOException { - ImageRecordReader recordReader = new ImageRecordReader(height,width,channels,labelMaker); - recordReader.initialize(split); - DataSetIterator iter = new RecordReaderDataSetIterator(recordReader, batchSize, 1, numClasses); - iter.setPreProcessor(new VGG16ImagePreProcessor()); - return iter; - } - - public static void downloadAndUntar() throws IOException { - File rootFile = new File(DATA_DIR); - if (!rootFile.exists()) { - rootFile.mkdir(); - } - File tarFile = new File(DATA_DIR, "flower_photos.tgz"); - if (!tarFile.isFile()) { - log.info("Downloading the flower dataset from "+DATA_URL+ "..."); - FileUtils.copyURLToFile( - new URL(DATA_URL), - tarFile); - } - ArchiveUtils.unzipFileTo(tarFile.getAbsolutePath(), rootFile.getAbsolutePath()); - } -} diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/transferlearning/vgg16/dataHelpers/FlowerDataSetIteratorFeaturized.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/transferlearning/vgg16/dataHelpers/FlowerDataSetIteratorFeaturized.java deleted file mode 100644 index fce588ca58..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/transferlearning/vgg16/dataHelpers/FlowerDataSetIteratorFeaturized.java +++ /dev/null @@ -1,62 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.transferlearning.vgg16.dataHelpers; - -import org.deeplearning4j.datasets.iterator.AsyncDataSetIterator; -import org.deeplearning4j.examples.multigpu.advanced.transferlearning.vgg16.FeaturizedPreSave; -import org.deeplearning4j.nn.modelimport.keras.exceptions.InvalidKerasConfigurationException; -import org.deeplearning4j.nn.modelimport.keras.exceptions.UnsupportedKerasConfigurationException; -import org.nd4j.linalg.dataset.ExistingMiniBatchDataSetIterator; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.slf4j.Logger; - -import java.io.File; -import java.io.IOException; - -/** - * Iterator for featurized data. - * @author susaneraly on 3/10/17. - */ -public class FlowerDataSetIteratorFeaturized { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(FlowerDataSetIteratorFeaturized.class); - static String featureExtractorLayer = FeaturizedPreSave.featurizeExtractionLayer; - - public static void setup(String featureExtractorLayerArg) { - featureExtractorLayer = featureExtractorLayerArg; - } - - public static DataSetIterator trainIterator() throws UnsupportedKerasConfigurationException, IOException, InvalidKerasConfigurationException { - runFeaturize(); - DataSetIterator existingTrainingData = new ExistingMiniBatchDataSetIterator(new File("trainFolder"),"flowers-"+featureExtractorLayer+"-train-%d.bin"); - DataSetIterator asyncTrainIter = new AsyncDataSetIterator(existingTrainingData); - return asyncTrainIter; - } - public static DataSetIterator testIterator() { - DataSetIterator existingTestData = new ExistingMiniBatchDataSetIterator(new File("testFolder"),"flowers-"+featureExtractorLayer+"-test-%d.bin"); - DataSetIterator asyncTestIter = new AsyncDataSetIterator(existingTestData); - return asyncTestIter; - } - - private static void runFeaturize() throws InvalidKerasConfigurationException, IOException, UnsupportedKerasConfigurationException { - File trainDir = new File("trainFolder","flowers-"+featureExtractorLayer+"-train-0.bin"); - if (!trainDir.isFile()) { - log.info("\n\tFEATURIZED DATA NOT FOUND. \n\t\tRUNNING \"FeaturizedPreSave\" first to do presave of featurized data"); - FeaturizedPreSave.main(null); - } - } -} diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/DataSetsBuilder.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/DataSetsBuilder.java deleted file mode 100644 index 88468dd027..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/DataSetsBuilder.java +++ /dev/null @@ -1,203 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.w2vsentiment; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; -import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.common.resources.Downloader; -import org.slf4j.Logger; - -import java.io.*; -import java.net.URL; -import java.util.Scanner; -import java.util.concurrent.atomic.AtomicInteger; - - -/** - * This class generates binary datasets out of raw text for further use on gpu for faster tuning sessions. - * Idea behind this approach is simple: if you're going to run over the same corpus over and over, it's more efficient - * to do all preprocessing once, and save datasets as binary files, instead of doing the same stuff over and over on the fly - * - */ -public class DataSetsBuilder { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(DataSetsBuilder.class); - - /** Data URL for downloading */ - public static final String DATA_URL = "http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz"; - /** Location to save and extract the training/testing data */ - public static final String DATA_PATH = FilenameUtils.concat(System.getProperty("java.io.tmpdir"), "dl4j_w2vSentiment/"); - /** Location (local file system) for the Google News vectors. Set this manually. */ - public static String wordVectorsPath = "/PATH/TO/YOUR/VECTORS/GoogleNews-vectors-negative300.bin.gz"; - - public static final String TRAIN_PATH = FilenameUtils.concat(System.getProperty("user.home"), "dl4j-examples-data/imdbpresaved/train"); - public static final String TEST_PATH = FilenameUtils.concat(System.getProperty("user.home"), "dl4j-examples-data/imdbpresaved/test"); - - @Parameter(names = {"-b","--batch"}, description = "BatchSize") - private int batchSize = 64; - - @Parameter(names = {"-l","--length"}, description = "Truncate max review length to") - private int truncateReviewsToLength = 256; - - public void run(String[] args) throws Exception { - JCommander jcmdr = new JCommander(this); - try { - jcmdr.parse(args); - } catch (ParameterException e) { - //User provides invalid input -> print the usage info - jcmdr.usage(); - try { - Thread.sleep(500); - } catch (Exception e2) { - } - System.exit(1); - } - - if (wordVectorsPath.startsWith("/PATH/TO/YOUR/VECTORS/")) { - System.out.println("wordVectorsPath has not been set. Checking default location in ~/dl4j-examples-data for download..."); - checkDownloadW2VECModel(); - } - //Download and extract data - downloadData(); - - WordVectors wordVectors = WordVectorSerializer.loadStaticModel(new File(wordVectorsPath)); - SentimentExampleIterator train = new SentimentExampleIterator(DATA_PATH, wordVectors, batchSize, truncateReviewsToLength, true); - SentimentExampleIterator test = new SentimentExampleIterator(DATA_PATH, wordVectors, batchSize, truncateReviewsToLength, false); - - log.info("Saving test data..."); - saveDatasets(test, TEST_PATH); - - log.info("Saving train data..."); - saveDatasets(train, TRAIN_PATH); - } - - private static void downloadData() throws Exception { - //Create directory if required - File directory = new File(DATA_PATH); - if(!directory.exists()) directory.mkdir(); - - //Download file: - String archizePath = DATA_PATH + "aclImdb_v1.tar.gz"; - File archiveFile = new File(archizePath); - String extractedPath = DATA_PATH + "aclImdb"; - File extractedFile = new File(extractedPath); - - if( !archiveFile.exists() ){ - System.out.println("Starting data download (80MB)..."); - FileUtils.copyURLToFile(new URL(DATA_URL), archiveFile); - System.out.println("Data (.tar.gz file) downloaded to " + archiveFile.getAbsolutePath()); - //Extract tar.gz file to output directory - extractTarGz(archizePath, DATA_PATH); - } else { - //Assume if archive (.tar.gz) exists, then data has already been extracted - System.out.println("Data (.tar.gz file) already exists at " + archiveFile.getAbsolutePath()); - if( !extractedFile.exists()){ - //Extract tar.gz file to output directory - extractTarGz(archizePath, DATA_PATH); - } else { - System.out.println("Data (extracted) already exists at " + extractedFile.getAbsolutePath()); - } - } - } - - private static final int BUFFER_SIZE = 4096; - private static void extractTarGz(String filePath, String outputPath) throws IOException { - int fileCount = 0; - int dirCount = 0; - System.out.print("Extracting files"); - try(TarArchiveInputStream tais = new TarArchiveInputStream( - new GzipCompressorInputStream( new BufferedInputStream( new FileInputStream(filePath))))){ - TarArchiveEntry entry; - - /** Read the tar entries using the getNextEntry method **/ - while ((entry = (TarArchiveEntry) tais.getNextEntry()) != null) { - //System.out.println("Extracting file: " + entry.getName()); - - //Create directories as required - if (entry.isDirectory()) { - new File(outputPath + entry.getName()).mkdirs(); - dirCount++; - }else { - int count; - byte data[] = new byte[BUFFER_SIZE]; - - FileOutputStream fos = new FileOutputStream(outputPath + entry.getName()); - BufferedOutputStream dest = new BufferedOutputStream(fos,BUFFER_SIZE); - while ((count = tais.read(data, 0, BUFFER_SIZE)) != -1) { - dest.write(data, 0, count); - } - dest.close(); - fileCount++; - } - if(fileCount % 1000 == 0) System.out.print("."); - } - } - - System.out.println("\n" + fileCount + " files and " + dirCount + " directories extracted to: " + outputPath); - } - - protected void saveDatasets(DataSetIterator iterator, String dir) { - AtomicInteger counter = new AtomicInteger(0); - new File(dir).mkdirs(); - while (iterator.hasNext()) { - String path = FilenameUtils.concat(dir, "dataset-" + (counter.getAndIncrement()) + ".bin"); - iterator.next().save(new File(path)); - - if (counter.get() % 500 == 0) - log.info("{} datasets saved so far...", counter.get()); - } - } - - - public static void checkDownloadW2VECModel() throws IOException { - String defaultwordVectorsPath = FilenameUtils.concat(System.getProperty("user.home"), "dl4j-examples-data/w2vec300"); - String md5w2vec = "1c892c4707a8a1a508b01a01735c0339"; - wordVectorsPath = new File(defaultwordVectorsPath, "GoogleNews-vectors-negative300.bin.gz").getAbsolutePath(); - if (new File(wordVectorsPath).exists()) { - System.out.println("\n\tGoogleNews-vectors-negative300.bin.gz file found at path: " + defaultwordVectorsPath); - System.out.println("\tChecking md5 of existing file.."); - if (Downloader.checkMD5OfFile(md5w2vec, new File(wordVectorsPath))) { - System.out.println("\tExisting file hash matches."); - return; - } else { - System.out.println("\tExisting file hash doesn't match. Retrying download..."); - } - } else { - System.out.println("\n\tNo previous download of GoogleNews-vectors-negative300.bin.gz found at path: " + defaultwordVectorsPath); - } - System.out.println("\tWARNING: GoogleNews-vectors-negative300.bin.gz is a 1.5GB file."); - System.out.println("\tPress \"ENTER\" to start a download of GoogleNews-vectors-negative300.bin.gz to " + defaultwordVectorsPath); - Scanner scanner = new Scanner(System.in); - scanner.nextLine(); - System.out.println("Starting model download (1.5GB!)..."); - Downloader.download("Word2Vec", new URL("https://dl4jdata.blob.core.windows.net/resources/wordvectors/GoogleNews-vectors-negative300.bin.gz"), new File(wordVectorsPath), md5w2vec, 5); - System.out.println("Successfully downloaded word2vec model to " + wordVectorsPath); - } - - public static void main(String[] args) throws Exception { - new DataSetsBuilder().run(args); - } -} diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/ImdbReviewClassificationRNN.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/ImdbReviewClassificationRNN.java deleted file mode 100644 index 8a7dc9fc21..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/ImdbReviewClassificationRNN.java +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.w2vsentiment; - -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.LSTM; -import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.PerformanceListener; -import org.deeplearning4j.parallelism.ParallelWrapper; -import org.nd4j.evaluation.classification.Evaluation; -import org.nd4j.jita.conf.CudaEnvironment; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.dataset.ExistingMiniBatchDataSetIterator; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.Adam; -import org.nd4j.linalg.lossfunctions.LossFunctions; -import org.slf4j.Logger; - -import java.io.File; - -import static org.deeplearning4j.examples.multigpu.advanced.w2vsentiment.DataSetsBuilder.TEST_PATH; -import static org.deeplearning4j.examples.multigpu.advanced.w2vsentiment.DataSetsBuilder.TRAIN_PATH; - -/** - * Example: Given a movie review (raw text), classify that movie review as either positive or negative based on the words it contains. - * This example is the multi-gpu version of the dl4j-example example of the same name. - * - * Here the dataset is presaved to save time on multiple epochs. - * @author Alex Black - */ -public class ImdbReviewClassificationRNN { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ImdbReviewClassificationRNN.class); - - public static void main(String[] args) throws Exception { - - int vectorSize = 300; //Size of the word vectors. 300 in the Google News model - int nEpochs = 1; //Number of epochs (full passes of training data) to train on - -// Nd4j.setDataType(DataBuffer.Type.DOUBLE); - - CudaEnvironment.getInstance().getConfiguration() - // key option enabled - .allowMultiGPU(true) - - // we're allowing larger memory caches - .setMaximumDeviceCache(2L * 1024L * 1024L * 1024L) - - // cross-device access is used for faster model averaging over pcie - .allowCrossDeviceAccess(true); - - //Set up network configuration - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .updater(new Adam.Builder().learningRate(2e-2).build()) - .l2(1e-5) - .weightInit(WeightInit.XAVIER) - .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue).gradientNormalizationThreshold(1.0) - .list() - .layer(0, new LSTM.Builder().nIn(vectorSize).nOut(256) - .activation(Activation.TANH).build()) - .layer(1, new RnnOutputLayer.Builder().activation(Activation.SOFTMAX) - .lossFunction(LossFunctions.LossFunction.MCXENT).nIn(256).nOut(2).build()) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - net.setListeners(new PerformanceListener(10, true)); - - if (!new File(TRAIN_PATH).exists() || !new File(TEST_PATH).exists()) { - new DataSetsBuilder().run(args); - } - //DataSetIterators for training and testing respectively - DataSetIterator train = new ExistingMiniBatchDataSetIterator(new File(TRAIN_PATH)); - DataSetIterator test = new ExistingMiniBatchDataSetIterator(new File(TEST_PATH)); - - ParallelWrapper pw = new ParallelWrapper.Builder<>(net) - .prefetchBuffer(16 * Nd4j.getAffinityManager().getNumberOfDevices()) - .reportScoreAfterAveraging(true) - .averagingFrequency(10) - .workers(Nd4j.getAffinityManager().getNumberOfDevices()) - .build(); - - log.info("Starting training..."); - for (int i = 0; i < nEpochs; i++) { - pw.fit(train); - train.reset(); - } - - log.info("Starting evaluation..."); - - //Run evaluation. This is on 25k reviews, so can take some time - Evaluation evaluation = net.evaluate(test); - System.out.println(evaluation.stats()); - } - -} diff --git a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/SentimentExampleIterator.java b/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/SentimentExampleIterator.java deleted file mode 100644 index d63b44abef..0000000000 --- a/cuda-specific-examples/src/main/java/org/deeplearning4j/examples/multigpu/advanced/w2vsentiment/SentimentExampleIterator.java +++ /dev/null @@ -1,282 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.multigpu.advanced.w2vsentiment; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.deeplearning4j.models.embeddings.wordvectors.WordVectors; -import org.deeplearning4j.text.tokenization.tokenizer.preprocessor.CommonPreprocessor; -import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory; -import org.deeplearning4j.text.tokenization.tokenizerfactory.TokenizerFactory; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.DataSetPreProcessor; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.indexing.INDArrayIndex; -import org.nd4j.linalg.indexing.NDArrayIndex; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; - -/** This is a DataSetIterator that is specialized for the IMDB review dataset used in the Word2VecSentimentRNN example - * It takes either the train or test set data from this data set, plus a WordVectors object (typically the Google News - * 300 pretrained vectors from https://code.google.com/p/word2vec/) and generates training data sets.
- * Inputs/features: variable-length time series, where each word (with unknown words removed) is represented by - * its Word2Vec vector representation.
- * Labels/target: a single class (negative or positive), predicted at the final time step (word) of each review - * - * @author Alex Black - */ -public class SentimentExampleIterator implements DataSetIterator { - private final WordVectors wordVectors; - private final int batchSize; - private final int vectorSize; - private final int truncateLength; - - private int cursor = 0; - private final File[] positiveFiles; - private final File[] negativeFiles; - private final TokenizerFactory tokenizerFactory; - - /** - * @param dataDirectory the directory of the IMDB review data set - * @param wordVectors WordVectors object - * @param batchSize Size of each minibatch for training - * @param truncateLength If reviews exceed - * @param train If true: return the training data. If false: return the testing data. - */ - public SentimentExampleIterator(String dataDirectory, WordVectors wordVectors, int batchSize, int truncateLength, boolean train) throws IOException { - this.batchSize = batchSize; - this.vectorSize = wordVectors.getWordVector(wordVectors.vocab().wordAtIndex(0)).length; - - - File p = new File(FilenameUtils.concat(dataDirectory, "aclImdb/" + (train ? "train" : "test") + "/pos/") + "/"); - File n = new File(FilenameUtils.concat(dataDirectory, "aclImdb/" + (train ? "train" : "test") + "/neg/") + "/"); - positiveFiles = p.listFiles(); - negativeFiles = n.listFiles(); - - this.wordVectors = wordVectors; - this.truncateLength = truncateLength; - - tokenizerFactory = new DefaultTokenizerFactory(); - tokenizerFactory.setTokenPreProcessor(new CommonPreprocessor()); - } - - - @Override - public DataSet next(int num) { - if (cursor >= positiveFiles.length + negativeFiles.length) throw new NoSuchElementException(); - try{ - return nextDataSet(num); - }catch(IOException e){ - throw new RuntimeException(e); - } - } - - private DataSet nextDataSet(int num) throws IOException { - //First: load reviews to String. Alternate positive and negative reviews - List reviews = new ArrayList<>(num); - boolean[] positive = new boolean[num]; - for( int i=0; i> allTokens = new ArrayList<>(reviews.size()); - int maxLength = 0; - for(String s : reviews){ - List tokens = tokenizerFactory.create(s).getTokens(); - List tokensFiltered = new ArrayList<>(); - for(String t : tokens ){ - if(wordVectors.hasWord(t)) tokensFiltered.add(t); - } - allTokens.add(tokensFiltered); - maxLength = Math.max(maxLength,tokensFiltered.size()); - } - - //If longest review exceeds 'truncateLength': only take the first 'truncateLength' words - if(maxLength > truncateLength) maxLength = truncateLength; - - //Create data for training - //Here: we have reviews.size() examples of varying lengths - INDArray features = Nd4j.create(reviews.size(), vectorSize, maxLength); - INDArray labels = Nd4j.create(reviews.size(), 2, maxLength); //Two labels: positive or negative - //Because we are dealing with reviews of different lengths and only one output at the final time step: use padding arrays - //Mask arrays contain 1 if data is present at that time step for that example, or 0 if data is just padding - INDArray featuresMask = Nd4j.zeros(reviews.size(), maxLength); - INDArray labelsMask = Nd4j.zeros(reviews.size(), maxLength); - - int[] temp = new int[2]; - for( int i=0; i tokens = allTokens.get(i); - temp[0] = i; - //Get word vectors for each word in review, and put them in the training data - for( int j=0; j 1.0 in features mask - } - - int idx = (positive[i] ? 0 : 1); - int lastIdx = Math.min(tokens.size(),maxLength); - labels.putScalar(new int[]{i,idx,lastIdx-1},1.0); //Set label: [0,1] for negative, [1,0] for positive - labelsMask.putScalar(new int[]{i,lastIdx-1},1.0); //Specify that an output exists at the final time step for this example - } - - return new DataSet(features,labels,featuresMask,labelsMask); - } - - public int totalExamples() { - return positiveFiles.length + negativeFiles.length; - } - - @Override - public int inputColumns() { - return vectorSize; - } - - @Override - public int totalOutcomes() { - return 2; - } - - @Override - public void reset() { - cursor = 0; - } - - public boolean resetSupported() { - return true; - } - - @Override - public boolean asyncSupported() { - return true; - } - - @Override - public int batch() { - return batchSize; - } - - public int numExamples() { - return totalExamples(); - } - - @Override - public void setPreProcessor(DataSetPreProcessor preProcessor) { - throw new UnsupportedOperationException(); - } - - @Override - public List getLabels() { - return Arrays.asList("positive","negative"); - } - - @Override - public boolean hasNext() { - return cursor < numExamples(); - } - - @Override - public DataSet next() { - return next(batchSize); - } - - @Override - public void remove() { - - } - @Override - public DataSetPreProcessor getPreProcessor() { - throw new UnsupportedOperationException("Not implemented"); - } - - /** Convenience method for loading review to String */ - public String loadReviewToString(int index) throws IOException{ - File f; - if(index%2 == 0) f = positiveFiles[index/2]; - else f = negativeFiles[index/2]; - return FileUtils.readFileToString(f); - } - - /** Convenience method to get label for review */ - public boolean isPositiveReview(int index){ - return index%2 == 0; - } - - /** - * Used post training to load a review from a file to a features INDArray that can be passed to the network output method - * - * @param file File to load the review from - * @param maxLength Maximum length (if review is longer than this: truncate to maxLength). Use Integer.MAX_VALUE to not nruncate - * @return Features array - * @throws IOException If file cannot be read - */ - public INDArray loadFeaturesFromFile(File file, int maxLength) throws IOException { - String review = FileUtils.readFileToString(file); - return loadFeaturesFromString(review, maxLength); - } - - /** - * Used post training to convert a String to a features INDArray that can be passed to the network output method - * - * @param reviewContents Contents of the review to vectorize - * @param maxLength Maximum length (if review is longer than this: truncate to maxLength). Use Integer.MAX_VALUE to not nruncate - * @return Features array for the given input String - */ - public INDArray loadFeaturesFromString(String reviewContents, int maxLength){ - List tokens = tokenizerFactory.create(reviewContents).getTokens(); - List tokensFiltered = new ArrayList<>(); - for(String t : tokens ){ - if(wordVectors.hasWord(t)) tokensFiltered.add(t); - } - int outputLength = Math.max(maxLength,tokensFiltered.size()); - - INDArray features = Nd4j.create(1, vectorSize, outputLength); - - for( int j=0; j4.0.0 org.deeplearning4j - dl4j-distributed-training-examples - 1.0.0-beta7 - Introduction to Distributed Training with DL4J - A set of examples introducing distributed training with the DL4J framework + data-pipeline-examples + 1.0.0-M1.1 + Building a data pipeline prior to modeling + Loading raw data and processing it before training - 1.0.0-beta7 + 1.0.0-M1.1 - + nd4j-native 1.8 - bin - 2.11 3.6.1 3.3.1 1.4.0 2.4.3 - 1.0.23 - 1.0.13 1.1.7 - 1.27 - 2.7.4 - 2.0.0 - 2.1.0 - 1.11.109 - 2.5.1 - UTF-8 + 2.11 + 2.4.3 - - - - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-annotations - ${jackson.version} - - - com.fasterxml.jackson.module - jackson-module-scala_${scala.binary.version} - ${jackson.version} - - - com.fasterxml.jackson.datatype - jackson-datatype-jdk8 - ${jackson.version} - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${jackson.version} - + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + - - org.datavec - datavec-spark_${scala.binary.version} - 1.0.0-beta7_spark_2 - - - org.deeplearning4j - deeplearning4j-ui - ${dl4j-master.version} - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.42.Final - - - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - org.nd4j ${nd4j.backend} @@ -121,73 +66,147 @@ org.datavec - datavec-spark_${scala.binary.version} + datavec-api ${dl4j-master.version} - org.deeplearning4j - dl4j-spark_${scala.binary.version} + org.datavec + datavec-data-image ${dl4j-master.version} - org.deeplearning4j - dl4j-spark-parameterserver_${scala.binary.version} - ${dl4j-master.version} + org.datavec + datavec-spark_${scala.binary.version} + 1.0.0-SNAPSHOT - com.beust - jcommander - ${jcommander.version} + org.datavec + datavec-local + ${dl4j-master.version} - org.deeplearning4j - deeplearning4j-nlp + deeplearning4j-datasets ${dl4j-master.version} - org.datavec - datavec-local + org.deeplearning4j + deeplearning4j-core ${dl4j-master.version} org.deeplearning4j - deeplearning4j-zoo + deeplearning4j-ui ${dl4j-master.version} + + + net.jpountz.lz4 + lz4 + + - - - org.apache.hadoop - hadoop-azure - ${azure.hadoop.version} - - - com.microsoft.azure - azure-storage - ${azure.storage.version} + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + + + org.apache.hadoop + log4j + + + org.slf4j + slf4j-log4j12 + + - com.amazonaws - aws-java-sdk-emr - ${aws.sdk.version} - provided + io.netty + netty-all + 4.1.22.Final - com.amazonaws - aws-java-sdk-s3 - ${aws.sdk.version} - provided + ch.qos.logback + logback-classic + ${logback.version} - org.jsoup - jsoup - 1.10.2 + org.apache.httpcomponents + httpclient + 4.3.5 + + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + + + maven-enforcer-plugin + 1.0.1 + + + enforce-default + + enforce + + + + + [${maven.minimum.version},) + ********** Minimum Maven Version is ${maven.minimum.version}. Please upgrade Maven before continuing (run "mvn --version" to check). ********** + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${java.version} + ${java.version} + + + + com.lewisd + lint-maven-plugin + 0.0.11 + + true + + DuplicateDep + RedundantPluginVersion + + + + + + pom-lint + validate + + check + + + + org.codehaus.mojo exec-maven-plugin @@ -242,16 +261,34 @@ - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - ${java.version} - ${java.version} - - + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + com.lewisd + lint-maven-plugin + [0.0.11,) + + check + + + + + + + + + + + + diff --git a/data-pipeline-examples/src/main/java/org/deeplearning4j/datapipelineexamples/formats/hdfs/conversion/CSVtoMapFileConversion.java b/data-pipeline-examples/src/main/java/org/deeplearning4j/datapipelineexamples/formats/hdfs/conversion/CSVtoMapFileConversion.java deleted file mode 100644 index 8952872e03..0000000000 --- a/data-pipeline-examples/src/main/java/org/deeplearning4j/datapipelineexamples/formats/hdfs/conversion/CSVtoMapFileConversion.java +++ /dev/null @@ -1,73 +0,0 @@ -/* ***************************************************************************** - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.datapipelineexamples.formats.hdfs.conversion; - -import org.datavec.api.records.converter.RecordReaderConverter; -import org.datavec.api.records.reader.RecordReader; -import org.datavec.api.records.reader.impl.csv.CSVRecordReader; -import org.datavec.api.split.FileSplit; -import org.datavec.hadoop.records.reader.mapfile.MapFileRecordReader; -import org.datavec.hadoop.records.writer.mapfile.MapFileRecordWriter; -import org.deeplearning4j.datapipelineexamples.utils.DownloaderUtility; - -import java.io.File; -import java.util.Random; - -/** - * A simple example on how to convert a CSV (text format) to a Hadoop MapFile format. After conversion, you can use the - * MapFileRecordReader in place of the CSV reader for your code - * - * Why would you want to do this? - * 1. Performance: reading from a map file is much faster than reading from a CSV (parse once, plus reading binary - * format instead of text format) - * 2. Randomization: MapFileRecordReader supports randomization of iteration order - * - * Note that MapFileRecordReader/Writer are in datavec-hadoop package. You will also likely need hadoop-common. - * - * @author Alex Black - */ -public class CSVtoMapFileConversion { - - public static void main(String[] args) throws Exception { - - //Create CSV reader - File irisFile = new File(DownloaderUtility.IRISDATA.Download(),"iris.txt"); - RecordReader recordReader = new CSVRecordReader(); - recordReader.initialize(new FileSplit(irisFile)); - - //Create map file writer - File mapFileLoc = new File("mapFileOutputDir"); - MapFileRecordWriter writer = new MapFileRecordWriter(mapFileLoc); - - //Convert to MapFile binary format: - RecordReaderConverter.convert(recordReader, writer); - - - //Read back in from binary MapFile, random order: - Random rng = new Random(12345); - RecordReader mapFileReader = new MapFileRecordReader(rng); - mapFileReader.initialize(new FileSplit(mapFileLoc)); - - //Print out: - while(mapFileReader.hasNext()){ - System.out.println(mapFileReader.next()); - } - - } - -} diff --git a/dl4j-distributed-training-examples/pom.xml b/dl4j-distributed-training-examples/pom.xml index ee66dd72b8..f01191f2b8 100644 --- a/dl4j-distributed-training-examples/pom.xml +++ b/dl4j-distributed-training-examples/pom.xml @@ -9,7 +9,7 @@ ~ ~ Unless required by applicable law or agreed to in writing, software ~ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See theDenseNetBuilder + ~ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ~ License for the specific language governing permissions and limitations ~ under the License. ~ @@ -22,52 +22,57 @@ 4.0.0 org.deeplearning4j - dl4j-examples - 1.0.0-beta7 - Introduction to DL4J - A set of examples introducing the DL4J framework + dl4j-distributed-training-examples + 1.0.0-M1.1 + Introduction to Distributed Training with DL4J + A set of examples introducing distributed training with the DL4J framework - 1.0.0-beta7 + 1.0.0-M1.1 nd4j-native 1.8 - 3.6.1 + bin + 2.11 + 3.8.1 3.3.1 1.4.0 2.4.3 1.0.23 1.0.13 1.1.7 + 1.27 + 2.7.4 + 2.0.0 + 2.1.0 + 1.11.109 + 2.5.1 UTF-8 - - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.48.Final - - - + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + + + - org.deeplearning4j - dl4j-spark_2.11 - ${dl4j-master.version} - - - org.deeplearning4j - dl4j-spark-parameterserver_2.11 - ${dl4j-master.version} + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} org.nd4j @@ -76,139 +81,81 @@ org.datavec - datavec-api - ${dl4j-master.version} - - - org.datavec - datavec-data-image - ${dl4j-master.version} - - - org.datavec - datavec-local + datavec-spark_${scala.binary.version} ${dl4j-master.version} org.deeplearning4j - deeplearning4j-datasets + dl4j-spark_${scala.binary.version} ${dl4j-master.version} org.deeplearning4j - deeplearning4j-core + dl4j-spark-parameterserver_${scala.binary.version} ${dl4j-master.version} - org.deeplearning4j - deeplearning4j-ui - ${dl4j-master.version} + com.beust + jcommander + ${jcommander.version} + org.deeplearning4j - deeplearning4j-zoo + deeplearning4j-nlp ${dl4j-master.version} - org.deeplearning4j - deeplearning4j-parallel-wrapper + deeplearning4j-zoo ${dl4j-master.version} - - - jfree - jfreechart - ${jfreechart.version} - - - org.jfree - jcommon - ${jcommon.version} - - + - org.apache.httpcomponents - httpclient - 4.3.5 + org.apache.hadoop + hadoop-azure + ${azure.hadoop.version} + - ch.qos.logback - logback-classic - ${logback.version} + com.microsoft.azure + azure-storage + ${azure.storage.version} - org.datavec - datavec-data-codec - ${dl4j-master.version} + com.amazonaws + aws-java-sdk-emr + ${aws.sdk.version} + provided - org.bytedeco - javacv-platform - 1.5.3 + com.amazonaws + aws-java-sdk-s3 + ${aws.sdk.version} + provided org.jsoup jsoup - 1.12.1 + 1.10.2 - + - maven-enforcer-plugin - 1.0.1 - - - enforce-default - - enforce - - - - - [${maven.minimum.version},) - ********** Minimum Maven Version is ${maven.minimum.version}. Please upgrade Maven before continuing (run "mvn --version" to check). ********** - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven-compiler-plugin.version} - - ${java.version} - ${java.version} - - - - com.lewisd - lint-maven-plugin - 0.0.11 - - true - - DuplicateDep - RedundantPluginVersion - - - - - - pom-lint - validate - - check - - - + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + org.codehaus.mojo exec-maven-plugin @@ -263,34 +210,16 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + ${java.version} + ${java.version} + + - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - com.lewisd - lint-maven-plugin - [0.0.11,) - - check - - - - - - - - - - - - diff --git a/dl4j-distributed-training-examples/scripts/patentExamplePreproc.sh b/dl4j-distributed-training-examples/scripts/patentExamplePreproc.sh index 20aa4ad192..32d1591fde 100644 --- a/dl4j-distributed-training-examples/scripts/patentExamplePreproc.sh +++ b/dl4j-distributed-training-examples/scripts/patentExamplePreproc.sh @@ -32,7 +32,7 @@ MINIBATCH=32 #Minibatch size for preprocessed #Other variables. Don't modify these SCRIPTDIR=$(dirname "$0") -JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-beta7-bin.jar +JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar AZURE_ACCT=fs.azure.account.key.${AZURE_STORAGE_ACCT}.blob.core.windows.net diff --git a/dl4j-distributed-training-examples/scripts/patentExampleTrain.sh b/dl4j-distributed-training-examples/scripts/patentExampleTrain.sh index 9805dd3771..2710621bb3 100644 --- a/dl4j-distributed-training-examples/scripts/patentExampleTrain.sh +++ b/dl4j-distributed-training-examples/scripts/patentExampleTrain.sh @@ -40,7 +40,7 @@ AERON_BUFFER=33554432 #Other variables. Don't modify these SCRIPTDIR=$(dirname "$0") -JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-beta7-bin.jar +JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar AZURE_ACCT=fs.azure.account.key.${AZURE_STORAGE_ACCT}.blob.core.windows.net diff --git a/dl4j-distributed-training-examples/scripts/tinyImagenetTrain.sh b/dl4j-distributed-training-examples/scripts/tinyImagenetTrain.sh index 52b1552b8e..e58d21edb1 100644 --- a/dl4j-distributed-training-examples/scripts/tinyImagenetTrain.sh +++ b/dl4j-distributed-training-examples/scripts/tinyImagenetTrain.sh @@ -32,7 +32,7 @@ EXECUTOR_MEMORY=12G MASTER_PORT=7077 # Port for the spark master. Default is 7077 MINIBATCH=32 # Minibatch size for preprocessed datasets SCRIPTDIR=$(dirname "$0") -JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-beta7-bin.jar +JARFILE=${SCRIPTDIR}/../target/dl4j-spark-1.0.0-M1.1-bin.jar #Memory OFFHEAP_MEM_JAVACPP=20G # Maximum amount of off-heap memory diff --git a/dl4j-examples/README.md b/dl4j-examples/README.md index 7fe483ccce..38d65755c8 100644 --- a/dl4j-examples/README.md +++ b/dl4j-examples/README.md @@ -6,6 +6,8 @@ The pom file in this project can be used as a template for a user's own project. [Go back](../README.md) to the main repository page to explore other features/functionality of the **Eclipse Deeplearning4J** ecosystem. File an issue [here](https://github.com/eclipse/deeplearning4j-examples/issues) to request new features. + + ### QUICKSTART #### Modeling Examples diff --git a/dl4j-examples/pom.xml b/dl4j-examples/pom.xml index 97f1453ff3..caef85e881 100644 --- a/dl4j-examples/pom.xml +++ b/dl4j-examples/pom.xml @@ -23,17 +23,17 @@ org.deeplearning4j dl4j-examples - 1.0.0-beta7 + 1.0.0-SNAPSHOT Introduction to DL4J A set of examples introducing the DL4J framework - 1.0.0-beta7 + 1.0.0-M1.1 nd4j-native 1.8 - 3.6.1 + 3.8.1 3.3.1 1.4.0 2.4.3 @@ -41,22 +41,26 @@ 1.0.13 1.1.7 UTF-8 + 5.8.0-M1 + 1.5.5 + - - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.48.Final - - - + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + @@ -64,6 +68,8 @@ ${nd4j.backend} ${dl4j-master.version} + + org.datavec datavec-api @@ -127,20 +133,48 @@ logback-classic ${logback.version} - - org.datavec - datavec-data-codec - ${dl4j-master.version} - + org.bytedeco javacv-platform - 1.5.2 + ${javacv.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + maven-enforcer-plugin 1.0.1 @@ -278,4 +312,4 @@ - \ No newline at end of file + diff --git a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/features/tsne/TSNEStandardExample.java b/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/features/tsne/TSNEStandardExample.java deleted file mode 100644 index d45e97c85f..0000000000 --- a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/features/tsne/TSNEStandardExample.java +++ /dev/null @@ -1,95 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.quickstart.features.tsne; - -import org.deeplearning4j.models.embeddings.inmemory.InMemoryLookupTable; -import org.deeplearning4j.models.embeddings.loader.WordVectorSerializer; -import org.deeplearning4j.models.word2vec.wordstore.VocabCache; -import org.deeplearning4j.plot.BarnesHutTsne; -import org.deeplearning4j.examples.utils.DownloaderUtility; -import org.nd4j.linalg.api.buffer.DataType; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.common.primitives.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -/** - * Created by agibsonccc on 9/20/14. - *

- * Dimensionality reduction for high-dimension datasets - */ -public class TSNEStandardExample { - private static Logger log = LoggerFactory.getLogger(TSNEStandardExample.class); - public static String dataLocalPath; - - public static void main(String[] args) throws Exception { - dataLocalPath = DownloaderUtility.NLPDATA.Download(); - //STEP 1: Initialization - int iterations = 100; - //create an n-dimensional array of doubles - Nd4j.setDefaultDataTypes(DataType.DOUBLE, DataType.DOUBLE); - List cacheList = new ArrayList<>(); //cacheList is a dynamic array of strings used to hold all words - - //STEP 2: Turn text input into a list of words - log.info("Load & Vectorize data...."); - File wordFile = new File(dataLocalPath,"words.txt"); //Open the file - //Get the data of all unique word vectors - Pair vectors = WordVectorSerializer.loadTxt(wordFile); - VocabCache cache = vectors.getSecond(); - INDArray weights = vectors.getFirst().getSyn0(); //seperate weights of unique words into their own list - - for (int i = 0; i < cache.numWords(); i++) //seperate strings of words into their own list - cacheList.add(cache.wordAtIndex(i)); - - //STEP 3: build a dual-tree tsne to use later - log.info("Build model...."); - BarnesHutTsne tsne = new BarnesHutTsne.Builder() - .setMaxIter(iterations).theta(0.5) - .normalize(false) - .learningRate(500) - .useAdaGrad(false) -// .usePca(false) - .build(); - - //STEP 4: establish the tsne values and save them to a file - log.info("Store TSNE Coordinates for Plotting...."); - String outputFile = "target/archive-tmp/tsne-standard-coords.csv"; - (new File(outputFile)).getParentFile().mkdirs(); - - tsne.fit(weights); - tsne.saveAsFile(cacheList, outputFile); - //This tsne will use the weights of the vectors as its matrix, have two dimensions, use the words strings as - //labels, and be written to the outputFile created on the previous line - // Plot Data with gnuplot - // set datafile separator "," - // plot 'tsne-standard-coords.csv' using 1:2:3 with labels font "Times,8" - //!!! Possible error: plot was recently deprecated. Might need to re-do the last line - // - // If you use nDims=3 in the call to tsne.plot above, you can use the following gnuplot commands to - // generate a 3d visualization of the word vectors: - // set datafile separator "," - // splot 'tsne-standard-coords.csv' using 1:2:3:4 with labels font "Times,8" - } - - -} diff --git a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/MemorizeSequence.java b/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/MemorizeSequence.java index dd3061b8c9..1f4b48b418 100644 --- a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/MemorizeSequence.java +++ b/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/MemorizeSequence.java @@ -20,6 +20,7 @@ import org.deeplearning4j.nn.conf.MultiLayerConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration; import org.deeplearning4j.nn.conf.NeuralNetConfiguration.ListBuilder; +import org.deeplearning4j.nn.conf.Updater; import org.deeplearning4j.nn.conf.layers.LSTM; import org.deeplearning4j.nn.conf.layers.RnnOutputLayer; import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; @@ -27,7 +28,7 @@ import org.deeplearning4j.optimize.listeners.ScoreIterationListener; import org.nd4j.linalg.activations.Activation; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.indexaccum.IMax; +import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.learning.config.RmsProp; @@ -147,7 +148,7 @@ public static void main(String[] args) { // first process the last output of the network to a concrete // neuron, the neuron with the highest output has the highest // chance to get chosen - int sampledCharacterIdx = Nd4j.getExecutioner().exec(new IMax(output, 1)).getInt(0); + int sampledCharacterIdx = Nd4j.getExecutioner().exec(new ArgMax(output, 1))[0].getInt(0); // print the chosen output System.out.print(LEARNSTRING_CHARS_LIST.get(sampledCharacterIdx)); diff --git a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/VideoFrameClassifier.java b/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/VideoFrameClassifier.java deleted file mode 100644 index 27728c4a79..0000000000 --- a/dl4j-examples/src/main/java/org/deeplearning4j/examples/quickstart/modeling/recurrent/VideoFrameClassifier.java +++ /dev/null @@ -1,236 +0,0 @@ -/* ***************************************************************************** - * Copyright (c) 2020 Konduit K.K. - * Copyright (c) 2015-2019 Skymind, Inc. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0. - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - * SPDX-License-Identifier: Apache-2.0 - ******************************************************************************/ - -package org.deeplearning4j.examples.quickstart.modeling.recurrent; - -import org.apache.commons.io.FileUtils; -import org.datavec.api.conf.Configuration; -import org.datavec.api.records.reader.SequenceRecordReader; -import org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader; -import org.datavec.api.split.InputSplit; -import org.datavec.api.split.NumberedFileInputSplit; -import org.datavec.codec.reader.NativeCodecRecordReader; -import org.deeplearning4j.datasets.datavec.SequenceRecordReaderDataSetIterator; -import org.deeplearning4j.examples.utils.DownloaderUtility; -import org.deeplearning4j.nn.conf.BackpropType; -import org.deeplearning4j.nn.conf.GradientNormalization; -import org.deeplearning4j.nn.conf.MultiLayerConfiguration; -import org.deeplearning4j.nn.conf.NeuralNetConfiguration; -import org.deeplearning4j.nn.conf.layers.*; -import org.deeplearning4j.nn.conf.preprocessor.CnnToFeedForwardPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.FeedForwardToRnnPreProcessor; -import org.deeplearning4j.nn.conf.preprocessor.RnnToCnnPreProcessor; -import org.deeplearning4j.nn.multilayer.MultiLayerNetwork; -import org.deeplearning4j.nn.weights.WeightInit; -import org.deeplearning4j.optimize.listeners.ScoreIterationListener; -import org.nd4j.evaluation.classification.Evaluation; -import org.nd4j.linalg.activations.Activation; -import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.dataset.AsyncDataSetIterator; -import org.nd4j.linalg.dataset.DataSet; -import org.nd4j.linalg.dataset.api.DataSetPreProcessor; -import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; -import org.nd4j.linalg.factory.Nd4j; -import org.nd4j.linalg.learning.config.AdaGrad; -import org.nd4j.linalg.lossfunctions.LossFunctions; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; - -/** - * Example: Combine convolutional, max pooling, dense (feed forward) and recurrent (LSTM) layers to classify each - * frame of a video (using a generated/synthetic video data set) - * Specifically, each video contains a shape (randomly selected: circles, squares, lines, arcs) which persist for - * multiple frames (though move between frames) and may leave the frame. Each video contains multiple shapes which - * are shown for some random number of frames. - * The network needs to classify these shapes, even when the shape has left the frame. - * - * This example is somewhat contrived, but shows data import and network configuration for classifying video frames. - * The data for this example is automatically downloaded to: - * "~/dl4j-examples-data/dl4j-examples/video/videoshapesexample" - * ******************************************************* - * @author Alex Black - */ -public class VideoFrameClassifier { - - public static final int N_VIDEOS = 500; - public static final int V_WIDTH = 130; - public static final int V_HEIGHT = 130; - public static final int V_NFRAMES = 150; - - public static void main(String[] args) throws Exception { - - // Note that you will need to run with at least 7G off heap memory - // if you want to keep this batchsize and train the nn config specified - int miniBatchSize = 16; - - String dataDirectory = DownloaderUtility.VIDEOEXAMPLE.Download() + "/videoshapesexample/"; - - //Set up network architecture: - MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder() - .seed(12345) - .l2(0.001) //l2 regularization on all layers - .updater(new AdaGrad(0.04)) - .list() - .layer(new ConvolutionLayer.Builder(10, 10) - .nIn(3) //3 channels: RGB - .nOut(30) - .stride(4, 4) - .activation(Activation.RELU) - .weightInit(WeightInit.RELU) - .build()) //Output: (130-10+0)/4+1 = 31 -> 31*31*30 - .layer(new SubsamplingLayer.Builder(SubsamplingLayer.PoolingType.MAX) - .kernelSize(3, 3) - .stride(2, 2).build()) //(31-3+0)/2+1 = 15 - .layer(new ConvolutionLayer.Builder(3, 3) - .nIn(30) - .nOut(10) - .stride(2, 2) - .activation(Activation.RELU) - .weightInit(WeightInit.RELU) - .build()) //Output: (15-3+0)/2+1 = 7 -> 7*7*10 = 490 - .layer(new DenseLayer.Builder() - .activation(Activation.RELU) - .nIn(490) - .nOut(50) - .weightInit(WeightInit.RELU) - .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) - .gradientNormalizationThreshold(10) - .updater(new AdaGrad(0.01)) - .build()) - .layer(new LSTM.Builder() - .activation(Activation.TANH) - .nIn(50) - .nOut(50) - .weightInit(WeightInit.XAVIER) - .updater(new AdaGrad(0.008)) - .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) - .gradientNormalizationThreshold(10) - .build()) - .layer(new RnnOutputLayer.Builder(LossFunctions.LossFunction.MCXENT) - .activation(Activation.SOFTMAX) - .nIn(50) - .nOut(4) //4 possible shapes: circle, square, arc, line - .weightInit(WeightInit.XAVIER) - .gradientNormalization(GradientNormalization.ClipElementWiseAbsoluteValue) - .gradientNormalizationThreshold(10) - .build()) - .inputPreProcessor(0, new RnnToCnnPreProcessor(V_HEIGHT, V_WIDTH, 3)) - .inputPreProcessor(3, new CnnToFeedForwardPreProcessor(7, 7, 10)) - .inputPreProcessor(4, new FeedForwardToRnnPreProcessor()) - .backpropType(BackpropType.TruncatedBPTT) - .tBPTTForwardLength(V_NFRAMES / 5) - .tBPTTBackwardLength(V_NFRAMES / 5) - .build(); - - MultiLayerNetwork net = new MultiLayerNetwork(conf); - net.init(); - - // summary of layer and parameters - System.out.println(net.summary()); - - int testStartIdx = (int) (0.9 * N_VIDEOS); //90% in train, 10% in test - int nTest = N_VIDEOS - testStartIdx; - - //Conduct learning - System.out.println("Starting training..."); - net.setListeners(new ScoreIterationListener(1)); - - int nTrainEpochs = 15; - for (int i = 0; i < nTrainEpochs; i++) { - DataSetIterator trainData = getDataSetIterator(dataDirectory, 0, testStartIdx - 1, miniBatchSize); - while(trainData.hasNext()) - net.fit(trainData.next()); - Nd4j.saveBinary(net.params(),new File("videomodel.bin")); - FileUtils.writeStringToFile(new File("videoconf.json"), conf.toJson(), (Charset) null); - System.out.println("Epoch " + i + " complete"); - - //Evaluate classification performance: - evaluatePerformance(net,testStartIdx,nTest,dataDirectory); - } - } - - - private static void evaluatePerformance(MultiLayerNetwork net, int testStartIdx, int nExamples, String outputDirectory) throws Exception { - //Assuming here that the full test data set doesn't fit in memory -> load 10 examples at a time - Map labelMap = new HashMap<>(); - labelMap.put(0, "circle"); - labelMap.put(1, "square"); - labelMap.put(2, "arc"); - labelMap.put(3, "line"); - Evaluation evaluation = new Evaluation(labelMap); - - DataSetIterator testData = getDataSetIterator(outputDirectory, testStartIdx, nExamples, 10); - while(testData.hasNext()) { - DataSet dsTest = testData.next(); - INDArray predicted = net.output(dsTest.getFeatures(), false); - INDArray actual = dsTest.getLabels(); - evaluation.evalTimeSeries(actual, predicted); - } - - System.out.println(evaluation.stats()); - } - - private static DataSetIterator getDataSetIterator(String dataDirectory, int startIdx, int nExamples, int miniBatchSize) throws Exception { - //Here, our data and labels are in separate files - //videos: shapes_0.mp4, shapes_1.mp4, etc - //labels: shapes_0.txt, shapes_1.txt, etc. One time step per line - - SequenceRecordReader featuresTrain = getFeaturesReader(dataDirectory, startIdx, nExamples); - SequenceRecordReader labelsTrain = getLabelsReader(dataDirectory, startIdx, nExamples); - - SequenceRecordReaderDataSetIterator sequenceIter = - new SequenceRecordReaderDataSetIterator(featuresTrain, labelsTrain, miniBatchSize, 4, false); - sequenceIter.setPreProcessor(new VideoPreProcessor()); - - //AsyncDataSetIterator: Used to (pre-load) load data in a separate thread - return new AsyncDataSetIterator(sequenceIter,1); - } - - private static SequenceRecordReader getFeaturesReader(String path, int startIdx, int num) throws IOException, InterruptedException { - //InputSplit is used here to define what the file paths look like - InputSplit is = new NumberedFileInputSplit(path + "shapes_%d.mp4", startIdx, startIdx + num - 1); - - Configuration conf = new Configuration(); - conf.set(NativeCodecRecordReader.RAVEL, "true"); - conf.set(NativeCodecRecordReader.START_FRAME, "0"); - conf.set(NativeCodecRecordReader.TOTAL_FRAMES, String.valueOf(V_NFRAMES)); - conf.set(NativeCodecRecordReader.ROWS, String.valueOf(V_WIDTH)); - conf.set(NativeCodecRecordReader.COLUMNS, String.valueOf(V_HEIGHT)); - NativeCodecRecordReader crr = new NativeCodecRecordReader(); - crr.initialize(conf, is); - return crr; - - } - - private static SequenceRecordReader getLabelsReader(String path, int startIdx, int num) throws Exception { - InputSplit isLabels = new NumberedFileInputSplit(path + "shapes_%d.txt", startIdx, startIdx + num - 1); - CSVSequenceRecordReader csvSeq = new CSVSequenceRecordReader(); - csvSeq.initialize(isLabels); - return csvSeq; - } - - private static class VideoPreProcessor implements DataSetPreProcessor { - @Override - public void preProcess(org.nd4j.linalg.dataset.api.DataSet toPreProcess) { - toPreProcess.getFeatures().divi(255); //[0,255] -> [0,1] for input pixel values - } - } -} diff --git a/dl4j-examples/src/main/java/org/deeplearning4j/examples/utils/PlotUtil.java b/dl4j-examples/src/main/java/org/deeplearning4j/examples/utils/PlotUtil.java index 892664ef9f..00c12f6b90 100644 --- a/dl4j-examples/src/main/java/org/deeplearning4j/examples/utils/PlotUtil.java +++ b/dl4j-examples/src/main/java/org/deeplearning4j/examples/utils/PlotUtil.java @@ -36,7 +36,7 @@ import org.jfree.ui.RectangleEdge; import org.jfree.ui.RectangleInsets; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.indexaccum.IMax; +import org.nd4j.linalg.api.ops.impl.indexaccum.custom.ArgMax; import org.nd4j.linalg.dataset.DataSet; import org.nd4j.linalg.dataset.api.iterator.DataSetIterator; import org.nd4j.linalg.factory.Nd4j; @@ -142,7 +142,7 @@ private static XYDataset createDataSetTrain(INDArray features, INDArray labels) XYSeries[] series = new XYSeries[nClasses]; for (int i = 0; i < series.length; i++) series[i] = new XYSeries("Class " + i); - INDArray argMax = Nd4j.getExecutioner().exec(new IMax(labels, 1)); + INDArray argMax = Nd4j.getExecutioner().exec(new ArgMax(labels, 1))[0]; for (int i = 0; i < nRows; i++) { int classIdx = (int) argMax.getDouble(i); series[classIdx].add(features.getDouble(i, 0), features.getDouble(i, 1)); diff --git a/dl4j-examples/src/test/java/org/deeplearning4j/examples/QuickTest.java b/dl4j-examples/src/test/java/org/deeplearning4j/examples/QuickTest.java new file mode 100644 index 0000000000..3e42a1abb9 --- /dev/null +++ b/dl4j-examples/src/test/java/org/deeplearning4j/examples/QuickTest.java @@ -0,0 +1,19 @@ +package org.deeplearning4j.examples; + +import org.deeplearning4j.examples.quickstart.modeling.convolution.LeNetMNIST; +import org.deeplearning4j.examples.quickstart.modeling.variationalautoencoder.VaeMNISTAnomaly; +import org.junit.jupiter.api.Test; + +public class QuickTest { + @Test + public void test() throws Exception { + System.out.println("Beginning LenetMNIST"); + LeNetMNIST.main(new String[]{}); + System.out.println("Ending LenetMNIST"); + System.out.println("Beginning VaeMNISTAnomaly"); + VaeMNISTAnomaly.main(new String[]{}); + System.out.println("Beginning VaeMNISTAnomaly"); + + } + +} diff --git a/cuda-specific-examples/src/main/resources/log4j.properties b/dl4j-examples/src/test/resources/log4j.properties similarity index 96% rename from cuda-specific-examples/src/main/resources/log4j.properties rename to dl4j-examples/src/test/resources/log4j.properties index 8f9355eeea..2b21e31bfb 100644 --- a/cuda-specific-examples/src/main/resources/log4j.properties +++ b/dl4j-examples/src/test/resources/log4j.properties @@ -25,6 +25,7 @@ log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n log4j.appender.org.springframework=DEBUG log4j.appender.org.nd4j=INFO log4j.appender.org.canova=INFO +log4j.appender.org.datavec=INFO log4j.appender.org.deeplearning4j=INFO log4j.appender.opennlp.uima=OFF log4j.appender.org.apache.uima=OFF @@ -33,6 +34,7 @@ log4j.appender.org.cleartk=OFF log4j.logger.org.springframework=INFO log4j.logger.org.nd4j=INFO log4j.logger.org.canova=INFO +log4j.logger.org.datavec=INFO log4j.logger.org.deeplearning4j=INFO log4j.logger.opennlp.uima.util=OFF log4j.logger.org.apache.uima=OFF diff --git a/cuda-specific-examples/src/main/resources/logback.xml b/dl4j-examples/src/test/resources/logback.xml similarity index 97% rename from cuda-specific-examples/src/main/resources/logback.xml rename to dl4j-examples/src/test/resources/logback.xml index 726fc1d406..43af9b0041 100644 --- a/cuda-specific-examples/src/main/resources/logback.xml +++ b/dl4j-examples/src/test/resources/logback.xml @@ -38,6 +38,7 @@ + diff --git a/mvn-project-template/pom.xml b/mvn-project-template/pom.xml index a95c935556..399722aad3 100644 --- a/mvn-project-template/pom.xml +++ b/mvn-project-template/pom.xml @@ -24,11 +24,11 @@ org.deeplearning4j deeplearning4j-example-sample - 1.0.0-beta7 + 1.0.0-M1.1 - 1.0.0-beta7 + 1.0.0-M1.1 1.2.3 1.8 2.4.3 @@ -56,23 +56,11 @@ - + - + ch.qos.logback @@ -149,4 +137,4 @@ - \ No newline at end of file + diff --git a/nd4j-ndarray-examples/pom.xml b/nd4j-ndarray-examples/pom.xml index 5172d7c77a..4c3dcdd341 100644 --- a/nd4j-ndarray-examples/pom.xml +++ b/nd4j-ndarray-examples/pom.xml @@ -23,20 +23,40 @@ org.deeplearning4j nd4j-ndarray-examples - 1.0.0-beta7 + 1.0.0-SNAPSHOT ND4J Examples operating on ndarrays Working with NDArrays + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + + + - 1.0.0-beta7 + 1.0.0-M1.1 nd4j-native 1.8 - 3.6.1 + 3.8.1 3.3.1 1.1.7 UTF-8 + 5.8.0-M1 + 3.0.0-M5 + ${maven-surefire-plugin.version} + @@ -55,11 +75,41 @@ deeplearning4j-utility-iterators ${dl4j-master.version} + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + maven-enforcer-plugin 1.0.1 @@ -143,4 +193,4 @@ - \ No newline at end of file + diff --git a/nd4j-ndarray-examples/src/main/java/org/nd4j/examples/quickstart/Nd4jEx5_Accumulations.java b/nd4j-ndarray-examples/src/main/java/org/nd4j/examples/quickstart/Nd4jEx5_Accumulations.java index 81f85bd8fb..b8ce8a21ad 100644 --- a/nd4j-ndarray-examples/src/main/java/org/nd4j/examples/quickstart/Nd4jEx5_Accumulations.java +++ b/nd4j-ndarray-examples/src/main/java/org/nd4j/examples/quickstart/Nd4jEx5_Accumulations.java @@ -18,8 +18,6 @@ package org.nd4j.examples.quickstart; import org.nd4j.linalg.api.ndarray.INDArray; -import org.nd4j.linalg.api.ops.impl.indexaccum.IAMax; -import org.nd4j.linalg.api.ops.impl.indexaccum.IMin; import org.nd4j.linalg.factory.Nd4j; import java.util.Arrays; @@ -71,11 +69,6 @@ public static void main(String[] args){ INDArray avgAlong0 = originalArray.mean(0); INDArray stdevAlong0 = originalArray.std(0); - // Index Accumulation operations - // IAMax returns index of max value along specified dimension - INDArray idxOfMaxInEachColumn = Nd4j.getExecutioner().exec(new IAMax(originalArray, 0)); - INDArray idxOfMaxInEachRow = Nd4j.getExecutioner().exec(new IAMax(originalArray, 1)); - System.out.println("\n\n\n"); System.out.println("min along dimension 0: " + minAlong0); @@ -83,8 +76,6 @@ public static void main(String[] args){ System.out.println("sum along dimension 0: " + sumAlong0); System.out.println("avg along dimension 0: " + avgAlong0); System.out.println("stddev along dimension 0: " + stdevAlong0); - System.out.println("Index of max dimension 0: " + idxOfMaxInEachColumn); - System.out.println("Index of max dimension 1: " + idxOfMaxInEachRow); //If we had instead performed these along dimension 1, we would instead get a [3,1] array out @@ -99,11 +90,6 @@ public static void main(String[] args){ //Index accumulations return an integer value. INDArray argMaxAlongDim0 = Nd4j.argMax(originalArray,0); //Index of the max value, along dimension 0 System.out.println("\n\nargmax along dimension 0: " + argMaxAlongDim0); - INDArray argMinAlongDim0 = Nd4j.getExecutioner().exec(new IMin(originalArray, 0)); //Index of the min value, along dimension 0 - System.out.println("argmin along dimension 0: " + argMinAlongDim0); - - - } diff --git a/nd4j-ndarray-examples/src/test/java/org/nd4j/examples/QuickTest.java b/nd4j-ndarray-examples/src/test/java/org/nd4j/examples/QuickTest.java new file mode 100644 index 0000000000..20a4ef45be --- /dev/null +++ b/nd4j-ndarray-examples/src/test/java/org/nd4j/examples/QuickTest.java @@ -0,0 +1,25 @@ +package org.nd4j.examples; + +import org.junit.jupiter.api.Test; +import org.nd4j.examples.advanced.lowlevelmodeling.MultiClassLogitExample; +import org.nd4j.examples.advanced.memoryoptimization.WorkspacesExample; +import org.nd4j.examples.advanced.operations.CustomOpsExamples; + +public class QuickTest { + + + @Test + public void runTestExamples() throws Exception { + System.out.println("Beginning MultiClassLogitExample"); + MultiClassLogitExample.main(new String[]{}); + System.out.println("Ending WorkspacesExample"); + WorkspacesExample.main(new String[]{}); + System.out.println("Beginning WorkspacesExample"); + System.out.println("Ending MultiClassLogitExample"); + CustomOpsExamples.main(new String[]{}); + + + } + + +} diff --git a/nd4j-ndarray-examples/src/test/resources/log4j.properties b/nd4j-ndarray-examples/src/test/resources/log4j.properties new file mode 100644 index 0000000000..2b21e31bfb --- /dev/null +++ b/nd4j-ndarray-examples/src/test/resources/log4j.properties @@ -0,0 +1,42 @@ +################################################################################ +# Copyright (c) 2020 Konduit K.K. +# Copyright (c) 2015-2019 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + + +log4j.rootLogger=ERROR, Console +log4j.logger.play=DEBUG +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.nd4j=INFO +log4j.appender.org.canova=INFO +log4j.appender.org.datavec=INFO +log4j.appender.org.deeplearning4j=INFO +log4j.appender.opennlp.uima=OFF +log4j.appender.org.apache.uima=OFF +log4j.appender.org.cleartk=OFF + +log4j.logger.org.springframework=INFO +log4j.logger.org.nd4j=INFO +log4j.logger.org.canova=INFO +log4j.logger.org.datavec=INFO +log4j.logger.org.deeplearning4j=INFO +log4j.logger.opennlp.uima.util=OFF +log4j.logger.org.apache.uima=OFF +log4j.logger.org.cleartk=OFF + diff --git a/nd4j-ndarray-examples/src/test/resources/logback.xml b/nd4j-ndarray-examples/src/test/resources/logback.xml new file mode 100644 index 0000000000..43af9b0041 --- /dev/null +++ b/nd4j-ndarray-examples/src/test/resources/logback.xml @@ -0,0 +1,54 @@ + + + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + diff --git a/rl4j-examples/pom.xml b/rl4j-examples/pom.xml index 2ccf15f828..3ea86320b4 100644 --- a/rl4j-examples/pom.xml +++ b/rl4j-examples/pom.xml @@ -1,80 +1,97 @@ - - 4.0.0 org.deeplearning4j - nd4j-ndarray-examples - 1.0.0-beta7 - ND4J Examples operating on ndarrays - Working with NDArrays - + rl4j-examples + 1.0.0-M1.1 - 1.0.0-beta7 - - - nd4j-native + 1.0.0-M1.1 + 1.0.0-M1.1 + 1.1.7 1.8 - 3.6.1 + 3.8.1 3.3.1 - 1.1.7 + nd4j-native UTF-8 + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + + + + org.nd4j ${nd4j.backend} - ${dl4j-master.version} - - - ch.qos.logback - logback-classic - ${logback.version} - - - org.deeplearning4j - deeplearning4j-utility-iterators - ${dl4j-master.version} + ${nd4j.version} + org.deeplearning4j rl4j-core - ${dl4j-master.version} + ${rl4j.version} org.deeplearning4j rl4j-gym - ${dl4j-master.version} + ${rl4j.version} org.deeplearning4j rl4j-ale - ${dl4j-master.version} + ${rl4j.version} + + + + org.bytedeco + ale-platform + 0.6.0-1.5.2 + + + org.junit.jupiter + junit-jupiter-engine + 5.4.2 + test + + + ch.qos.logback + logback-classic + ${logback.version} - - + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + maven-enforcer-plugin 1.0.1 diff --git a/samediff-examples/pom.xml b/samediff-examples/pom.xml index 2580e64803..035e9485a6 100644 --- a/samediff-examples/pom.xml +++ b/samediff-examples/pom.xml @@ -23,12 +23,12 @@ org.deeplearning4j samediff-examples - 1.0.0-beta7 + 1.0.0-M1.1 Quickstart to SameDiff A set of beginner examples introducing the SameDiff framework - 1.0.0-beta7 + 1.0.0-M1.1 nd4j-native @@ -41,23 +41,31 @@ 1.0.13 1.1.7 UTF-8 + 5.8.0-M1 + + + + orgdeeplearning4j-1177 + https://oss.sonatype.org/content/repositories/orgdeeplearning4j-1177 + + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + + - - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.42.Final - - - @@ -117,11 +125,40 @@ logback-classic ${logback.version} + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + maven-enforcer-plugin 1.0.1 diff --git a/samediff-examples/src/test/java/org/nd4j/examples/samediff/QuickTest.java b/samediff-examples/src/test/java/org/nd4j/examples/samediff/QuickTest.java new file mode 100644 index 0000000000..e6183f2611 --- /dev/null +++ b/samediff-examples/src/test/java/org/nd4j/examples/samediff/QuickTest.java @@ -0,0 +1,28 @@ +package org.nd4j.examples.samediff; + +import org.junit.jupiter.api.Test; +import org.nd4j.examples.samediff.quickstart.basics.Ex1_SameDiff_Basics; +import org.nd4j.examples.samediff.quickstart.basics.Ex2_LinearRegression; +import org.nd4j.examples.samediff.quickstart.basics.Ex3_Variables; +import org.nd4j.examples.samediff.quickstart.modeling.MNISTCNN; + +public class QuickTest { + + @Test + public void test() throws Exception { + System.out.println("Begin running Ex1_SameDiff_Basics"); + Ex1_SameDiff_Basics.main(new String[]{}); + System.out.println("End running Ex1_SameDiff_Basics"); + System.out.println("Begin running Ex2_LinearRegression"); + Ex2_LinearRegression.main(new String[]{}); + System.out.println("End running Ex2_LinearRegression"); + System.out.println("Begin running Ex3_Variables"); + Ex3_Variables.main(new String[]{}); + System.out.println("End running Ex3_Variables"); + System.out.println("Begin running MNISTCNN"); + MNISTCNN.main(new String[]{}); + System.out.println("End running MNISTCNN"); + + } + +} diff --git a/tensorflow-keras-import-examples/README.md b/tensorflow-keras-import-examples/README.md index cfd0861e45..8d1e355be3 100644 --- a/tensorflow-keras-import-examples/README.md +++ b/tensorflow-keras-import-examples/README.md @@ -1,6 +1,6 @@ ## Eclipse Deeplearning4j: Model Import -The DL4J supports models created in the popular Python Tensorflow and Keras frameworks. As of 1.0.0-beta7, Keras models (including tf.keras) can be imported into Deeplearning. TensorFlow frozen format models can be imported into SameDiff. +The DL4J supports models created in the popular Python Tensorflow and Keras frameworks. As of 1.0.0-M1.1, Keras models (including tf.keras) can be imported into Deeplearning. TensorFlow frozen format models can be imported into SameDiff. Models in Tensorflow have to be converted to "frozen" pbs (protobuf). More information on freezing Tensorflow models can be found [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py#L15) for Tensorflow 1.X and [here](https://leimao.github.io/blog/Save-Load-Inference-From-TF2-Frozen-Graph/) for Tensorflow 2.X. Keras models have to be saved in h5 format. More information can be found [here](https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model). Importing both Keras 1 and Keras 2 models are supported. Of note - importing models saved with tf.keras is also supported. Currently general TensorFlow operations within Keras models (i.e., those not part of the tf.keras API) are currently importable but support inference only. Full training is supported for anything that is part of the Keras API. diff --git a/tensorflow-keras-import-examples/ops-added-old.txt b/tensorflow-keras-import-examples/ops-added-old.txt new file mode 100644 index 0000000000..f605f17494 --- /dev/null +++ b/tensorflow-keras-import-examples/ops-added-old.txt @@ -0,0 +1,20 @@ +Placeholder,input +Const,Variable +Const,Variable_1 +Const,Variable_2 +Const,Variable_3 +Const,Variable_4 +Const,Variable_5 +Identity,Variable/read +Identity,Variable_1/read +Identity,Variable_2/read +Identity,Variable_3/read +Identity,Variable_4/read +Identity,Variable_5/read +MatMul,MatMul +Add,Add +MatMul,MatMul_1 +Add,Add_1 +MatMul,MatMul_2 +Add,add +Softmax,output diff --git a/tensorflow-keras-import-examples/ops-imported-old.txt b/tensorflow-keras-import-examples/ops-imported-old.txt new file mode 100644 index 0000000000..9ffe108d48 --- /dev/null +++ b/tensorflow-keras-import-examples/ops-imported-old.txt @@ -0,0 +1,13 @@ +Identity,Variable/read +Identity,Variable_1/read +Identity,Variable_2/read +Identity,Variable_3/read +Identity,Variable_4/read +Identity,Variable_5/read +MatMul,MatMul +Add,Add +MatMul,MatMul_1 +Add,Add_1 +MatMul,MatMul_2 +Add,add +Softmax,output diff --git a/tensorflow-keras-import-examples/ops-removed-old.txt b/tensorflow-keras-import-examples/ops-removed-old.txt new file mode 100644 index 0000000000..334669aa85 --- /dev/null +++ b/tensorflow-keras-import-examples/ops-removed-old.txt @@ -0,0 +1,20 @@ +input +Variable +Variable_1 +Variable_2 +Variable_3 +Variable_4 +Variable_5 +Variable/read +Variable_1/read +Variable_2/read +Variable_3/read +Variable_4/read +Variable_5/read +MatMul +Add +MatMul_1 +Add_1 +MatMul_2 +add +output diff --git a/tensorflow-keras-import-examples/pom.xml b/tensorflow-keras-import-examples/pom.xml index 04f7796cca..913fd6bad8 100644 --- a/tensorflow-keras-import-examples/pom.xml +++ b/tensorflow-keras-import-examples/pom.xml @@ -22,42 +22,43 @@ 4.0.0 org.deeplearning4j - samediff-examples - 1.0.0-beta7 - Quickstart to SameDiff - A set of beginner examples introducing the SameDiff framework + model-import-examples + 1.0.0-SNAPSHOT + model import examples + Loading models trained in keras or tensorflow - 1.0.0-beta7 + 1.0.0-M1.1 + nd4j-native - nd4j-native-platform 1.8 3.6.1 3.3.1 1.4.0 2.4.3 - 1.0.23 - 1.0.13 1.1.7 + 1.5.5 + 1.15.5 UTF-8 + 5.8.0-M1 + + + sonatype-nexus-snapshots + Sonatype Nexus Snapshots + https://oss.sonatype.org/content/repositories/snapshots + + false + + + true + daily + + + - - - - org.freemarker - freemarker - 2.3.29 - - - io.netty - netty-common - 4.1.42.Final - - - @@ -66,77 +67,76 @@ ${dl4j-master.version} - org.nd4j - nd4j-tensorflow + org.deeplearning4j + deeplearning4j-core ${dl4j-master.version} - + org.bytedeco + tensorflow-platform + ${tensorflow.version}-${javacpp.version} + - org.deeplearning4j - deeplearning4j-nlp - ${dl4j-master.version} + ch.qos.logback + logback-classic + ${logback.version} org.deeplearning4j deeplearning4j-zoo ${dl4j-master.version} + - org.deeplearning4j - deeplearning4j-datasets + org.nd4j + samediff-import-tensorflow ${dl4j-master.version} + + - org.deeplearning4j - deeplearning4j-core - ${dl4j-master.version} + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test - - - - - + - ch.qos.logback - logback-classic - ${logback.version} + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + maven-surefire-plugin + 3.0.0-M5 + true + + + org.apache.maven.surefire + surefire-junit-platform + 3.0.0-M5 + + + + maven-enforcer-plugin 1.0.1 diff --git a/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/bert/BertInferenceExample.java b/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/bert/BertInferenceExample.java index 47e6f1ab56..95191c0176 100644 --- a/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/bert/BertInferenceExample.java +++ b/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/bert/BertInferenceExample.java @@ -35,6 +35,8 @@ import org.nd4j.linalg.learning.config.Sgd; import org.nd4j.common.primitives.Pair; import org.nd4j.common.resources.Downloader; +import org.nd4j.samediff.frameworkimport.tensorflow.importer.TensorflowFrameworkImporter; + import java.io.File; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -56,7 +58,9 @@ *

* Similar to all tensorflow models the frozen pb is imported into a SameDiff graph. In the case of BERT the graph has to be * preprocessed (removing unneeded nodes etc) before inference can be carried out on it. More details with code below. + * @deprecated "Note this example uses an older TF import API that will be going away. Please consider using the newer import api." */ +@Deprecated() public class BertInferenceExample { public static String bertModelPath; diff --git a/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/mobilenet/ImportMobileNetExample.java b/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/mobilenet/ImportMobileNetExample.java index 1e54b18312..22139468d6 100644 --- a/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/mobilenet/ImportMobileNetExample.java +++ b/tensorflow-keras-import-examples/src/main/java/org/deeplearning4j/modelimportexamples/tf/advanced/mobilenet/ImportMobileNetExample.java @@ -9,10 +9,12 @@ import org.nd4j.linalg.api.ops.impl.image.ResizeBilinear; import org.nd4j.linalg.factory.Nd4j; import org.nd4j.linalg.indexing.NDArrayIndex; +import org.nd4j.samediff.frameworkimport.tensorflow.importer.TensorflowFrameworkImporter; import java.io.File; import java.io.IOException; import java.net.URL; +import java.util.Collections; /** * This example shows the ability to import and use Tensorflow models, specifically mobilenet, and use them for inference. @@ -24,8 +26,12 @@ public static void main(String[] args) throws Exception { // download and extract a tensorflow frozen model file (usually a .pb file) File modelFile = downloadModel(); + //new tensorflow import api + TensorflowFrameworkImporter tensorflowFrameworkImporter = new TensorflowFrameworkImporter(); + // import the frozen model into a SameDiff instance - SameDiff sd = SameDiff.importFrozenTF(modelFile); + //note: this uses the new tensorflow import api + SameDiff sd = tensorflowFrameworkImporter.runImport(modelFile.getAbsolutePath(), Collections.emptyMap()); System.out.println(sd.summary()); diff --git a/tensorflow-keras-import-examples/src/test/java/org/deeplearning4j/modelimportexamples/QuickTest.java b/tensorflow-keras-import-examples/src/test/java/org/deeplearning4j/modelimportexamples/QuickTest.java new file mode 100644 index 0000000000..ce9ddffa54 --- /dev/null +++ b/tensorflow-keras-import-examples/src/test/java/org/deeplearning4j/modelimportexamples/QuickTest.java @@ -0,0 +1,31 @@ +package org.deeplearning4j.modelimportexamples; + +import org.deeplearning4j.modelimportexamples.keras.quickstart.SimpleFunctionalMlpImport; +import org.deeplearning4j.modelimportexamples.keras.quickstart.SimpleSequentialMlpImport; +import org.deeplearning4j.modelimportexamples.tf.advanced.mobilenet.ImportMobileNetExample; +import org.deeplearning4j.modelimportexamples.tf.advanced.tfgraphrunnerinjava.TFGraphRunnerExample; +import org.deeplearning4j.modelimportexamples.tf.quickstart.MNISTMLP; +import org.junit.jupiter.api.Test; + +public class QuickTest { + + + @Test + public void runExamples() throws Exception { + System.out.println("Beginning MNISTMLP"); + MNISTMLP.main(new String[]{}); + System.out.println("Ended MNISTMLP"); + System.out.println("Beginning ImportMobileNetExample"); + ImportMobileNetExample.main(new String[]{}); + System.out.println("Ended ImportMobileNetExample"); + TFGraphRunnerExample.main(new String[]{}); + System.out.println("Beginning ImportMobileNetExample"); + SimpleFunctionalMlpImport.main(new String[]{}); + System.out.println("Ended ImportMobileNetExample"); + System.out.println("Beginning ImportMobileNetExample"); + SimpleSequentialMlpImport.main(new String[]{}); + System.out.println("Ended ImportMobileNetExample"); + + } + +} diff --git a/tensorflow-keras-import-examples/src/test/resources/log4j.properties b/tensorflow-keras-import-examples/src/test/resources/log4j.properties new file mode 100644 index 0000000000..2b21e31bfb --- /dev/null +++ b/tensorflow-keras-import-examples/src/test/resources/log4j.properties @@ -0,0 +1,42 @@ +################################################################################ +# Copyright (c) 2020 Konduit K.K. +# Copyright (c) 2015-2019 Skymind, Inc. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# SPDX-License-Identifier: Apache-2.0 +################################################################################ + + +log4j.rootLogger=ERROR, Console +log4j.logger.play=DEBUG +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ABSOLUTE} %-5p ~ %m%n + +log4j.appender.org.springframework=DEBUG +log4j.appender.org.nd4j=INFO +log4j.appender.org.canova=INFO +log4j.appender.org.datavec=INFO +log4j.appender.org.deeplearning4j=INFO +log4j.appender.opennlp.uima=OFF +log4j.appender.org.apache.uima=OFF +log4j.appender.org.cleartk=OFF + +log4j.logger.org.springframework=INFO +log4j.logger.org.nd4j=INFO +log4j.logger.org.canova=INFO +log4j.logger.org.datavec=INFO +log4j.logger.org.deeplearning4j=INFO +log4j.logger.opennlp.uima.util=OFF +log4j.logger.org.apache.uima=OFF +log4j.logger.org.cleartk=OFF + diff --git a/tensorflow-keras-import-examples/src/test/resources/logback.xml b/tensorflow-keras-import-examples/src/test/resources/logback.xml new file mode 100644 index 0000000000..43af9b0041 --- /dev/null +++ b/tensorflow-keras-import-examples/src/test/resources/logback.xml @@ -0,0 +1,54 @@ + + + + + + + + logs/application.log + + %date - [%level] - from %logger in %thread + %n%message%n%xException%n + + + + + + %logger{15} - %message%n%xException{5} + + + + + + + + + + + + + + + + + + + + + + diff --git a/tensorflow-keras-import-examples/variables-added-old.txt b/tensorflow-keras-import-examples/variables-added-old.txt new file mode 100644 index 0000000000..4963a74aa8 --- /dev/null +++ b/tensorflow-keras-import-examples/variables-added-old.txt @@ -0,0 +1,13 @@ +Variable/read,Variable/read +Variable_1/read,Variable_1/read +Variable_2/read,Variable_2/read +Variable_3/read,Variable_3/read +Variable_4/read,Variable_4/read +Variable_5/read,Variable_5/read +MatMul,MatMul +Add,Add +MatMul_1,MatMul_1 +Add_1,Add_1 +MatMul_2,MatMul_2 +add,add +output,output