Merge pull request #7 from equella/issue/6

#6 allows audio migrations

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/Config.java

@@ -21,6 +21,10 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 
 import org.apache.logging.log4j.LogManager;
@@ -47,6 +51,8 @@ public static enum ToolboxFunction {
 	public static final String KAL_SERVICE_URL = "migrate-to-kaltura.kal.service.url";
 	public static final String KAL_CATEGORIES = "migrate-to-kaltura.kal.categories";
 	public static final String OEQ_SEARCH_ATT_DESC = "migrate-to-kaltura.oeq.search.attachment.description";
+	public static final String OEQ_SEARCH_ATT_SUFFIXES_AUDIO = "migrate-to-kaltura.oeq.search.attachment.suffixes.audio";
+	public static final String OEQ_SEARCH_ATT_SUFFIXES_VIDEO = "migrate-to-kaltura.oeq.search.attachment.suffixes.video";
 	public static final String OEQ_SEARCH_KAL_TAGS_XPATH = "migrate-to-kaltura.oeq.search.kal.tags.xpath";
 	public static final String OEQ_KAL_ID = "migrate-to-kaltura.oeq.kal.id";
 	public static final String GENERAL_MAX_ITEMS_TO_MIGRATE = "migrate-to-kaltura.general.max.items.to.migrate";
@@ -67,13 +73,15 @@ public static enum ToolboxFunction {
 	public static final String GENERAL_DOWNLOAD_FOLDER = "general.download.folder";
 	public static final String GENERAL_DOWNLOAD_CHATTER = "general.download.chatter";
 	public static final String OEQ_SEARCH_API_REQUESTED_LENGTH = "oeq.search.api.requested.length";
-	
+
 	private static Logger LOGGER = LogManager.getLogger(Config.class);
 
 	private Properties store;
 	private boolean validConfig = true;
 	private String filepath;
 
+	private Map<String, List<String>> convertedCsvToLists = new HashMap<>();
+
 	// Meant for testing purposes
 	public Config(Properties props) {
 		store = props;
@@ -122,7 +130,6 @@ private void checkConfigsGeneral() {
 		checkConfig(OEQ_OAUTH_CLIENT_ID, true, true);
 		checkConfig(OEQ_OAUTH_CLIENT_SECRET, false, true);
 		checkConfig(OEQ_SEARCH_API, true, true);
-		checkConfig(OEQ_SEARCH_API, true, true);
 		checkConfig(GENERAL_OS_SLASH, true, true);
 		checkConfig(GENERAL_DOWNLOAD_FOLDER, true, true);
 		checkConfig(GENERAL_DOWNLOAD_CHATTER, true, true);		
@@ -150,6 +157,7 @@ private void checkConfigsMigrateToKaltura() {
 		checkConfig(OEQ_SEARCH_KAL_TAGS_XPATH, true, true);
 		checkConfig(OEQ_SEARCH_ATT_DESC, true, true);
 		checkConfig(OEQ_KAL_ID, true, true);
+		checkMigrateToKalturaAttachmentSuffixes();
 		checkConfig(GENERAL_MAX_ITEMS_TO_MIGRATE, true, false);
 		if(hasConfig(GENERAL_MAX_ITEMS_TO_MIGRATE)) {
 			checkInt(GENERAL_MAX_ITEMS_TO_MIGRATE, true);
@@ -162,7 +170,7 @@ private void checkConfigsMigrateToKaltura() {
 			}
 		}
 	}
-	
+
 	private void checkConfigsExportItems() {
 		checkConfig(EXPORT_ITEMS_OUTPUT_FILE, true, true);
 		if(validConfig) {
@@ -205,6 +213,10 @@ public String getConfig(String key) {
 		return store.getProperty(key);
 	}
 
+	public List<String> getConfigAsList(String key) {
+		return convertedCsvToLists.get(key);
+	}
+
 	/**
 	 *  Assumes checkInt(key) and isValidConfig() has already been called
 	 */
@@ -215,4 +227,42 @@ public int getConfigAsInt(String key) {
 	public boolean hasConfig(String key) {
 		return store.containsKey(key);
 	}
+
+	public void checkMigrateToKalturaAttachmentSuffixes() {
+		checkConfig(OEQ_SEARCH_ATT_SUFFIXES_AUDIO, true, true);
+		checkConfig(OEQ_SEARCH_ATT_SUFFIXES_VIDEO, true, true);
+
+		if(validConfig) {
+			// The script needs to know what type of media the attachment to upload is.  
+			// To ensure an unambiguous decision, the audio suffixes and video suffixes must be unique.
+
+			List<String> audios = Arrays.asList(getConfig(OEQ_SEARCH_ATT_SUFFIXES_AUDIO).toUpperCase().split(","));
+			List<String> videos = Arrays.asList(getConfig(OEQ_SEARCH_ATT_SUFFIXES_VIDEO).toUpperCase().split(","));
+
+			if(audios.size() != 0 && videos.size() != 0) {
+				for(String a : audios) {
+					if(a.trim().length() != 0) {
+						if(videos.contains(a.trim())) {
+							LOGGER.info("{} and {} must have distinct (case insensitive) sets.  Found [{}] in [{}].", OEQ_SEARCH_ATT_SUFFIXES_AUDIO, OEQ_SEARCH_ATT_SUFFIXES_VIDEO, a, getConfig(OEQ_SEARCH_ATT_SUFFIXES_VIDEO));
+							validConfig = false;
+							return;
+						}
+					}
+				}
+
+				for(String v : videos) {
+					if(v.trim().length() != 0) {
+						if(audios.contains(v.trim())) {
+							LOGGER.info("{} and {} must have distinct (case insensitive) sets.  Found [{}] in [{}].", OEQ_SEARCH_ATT_SUFFIXES_AUDIO, OEQ_SEARCH_ATT_SUFFIXES_VIDEO, v, getConfig(OEQ_SEARCH_ATT_SUFFIXES_AUDIO));
+							validConfig = false;
+							return;
+						}
+					}
+				}
+			}
+
+			convertedCsvToLists.put(OEQ_SEARCH_ATT_SUFFIXES_AUDIO, audios);
+			convertedCsvToLists.put(OEQ_SEARCH_ATT_SUFFIXES_VIDEO, videos);
+		}
+	}
 }

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/MigrateItemsToKalturaDriver.java

@@ -82,7 +82,7 @@ private void processItem(KalturaUtils ku, OpenEquellaRestUtils oeru, EquellaItem
 		kalturaUploadInProgress = true;
 		if(oeru.downloadAttachmentForKaltura(eqResource)) {	
 			// Add the video to Kaltura
-			ku.addVideo(eqResource, new OnCompletion<MediaEntry>() {
+			ku.addMedia(eqResource, new OnCompletion<MediaEntry>() {
 
 				@Override
 				public void onComplete(MediaEntry result) {

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/api/KalturaUtils.java

@@ -73,15 +73,22 @@ public boolean obtainSession(Config c) {
 		}	
 	}
 
-	public void addVideo(EquellaItem eItem, final OnCompletion<MediaEntry> onCompletion)
+	public void addMedia(EquellaItem eItem, final OnCompletion<MediaEntry> onCompletion)
 	{
 		LOGGER.info("{} - Processing resource:  ", eItem.getSignature());
 		MediaEntry entry = new MediaEntry();
 		entry.setName(eItem.getName());
 		entry.setDescription(eItem.getDescription());
 		entry.setTags(eItem.getKalturaTags());
 		entry.setCategories(appConfig.getConfig(Config.KAL_CATEGORIES));
-		entry.setMediaType(MediaType.VIDEO);
+		if(appConfig.getConfigAsList(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO).contains(eItem.getPrimaryFileType())) {
+			entry.setMediaType(MediaType.AUDIO);
+		} else {
+			// Precondition: the check of the file suffix being in the audio or video CSV has already happened.
+			// Assumes video type
+			entry.setMediaType(MediaType.VIDEO);
+		}
+
 		entry.setReferenceId(getUniqueString());
 
 		File fileToUpload = new File(eItem.getFilepath());

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/api/OpenEquellaRestUtils.java

@@ -336,6 +336,7 @@ public boolean downloadAttachmentForKaltura(EquellaItem ei) {
 		int bestFitAttIndex = -1;
 		String bestFitAttLink = "";
 		String bestFitAttFilename = "";
+		String bestFitFileSuffix = "";
 		long bestFitAttFilesize = -1;
 		JSONArray atts = null;
 
@@ -357,11 +358,16 @@ public boolean downloadAttachmentForKaltura(EquellaItem ei) {
 								ei.getSignature(), config.getConfig(Config.OEQ_SEARCH_ATT_DESC), att);
 						continue;
 					}
-
 					String filename = confirmAndGatherString(att, KEY_ATT_FILENAME);
-					if (!filename.toUpperCase().endsWith(".MP4") && !filename.toUpperCase().endsWith(".MOV")) {
+					String validSuffix = FileUtils.extractSuffix(config, filename);
+
+					if (validSuffix == null) {
 						LOGGER.info(
-								"{} - Not an attachment to migrate - filename doesn't end with [.MP4] or [.MOV]: {}", ei.getSignature(), att);
+								"{} - Not an attachment to migrate - filename doesn't end with one of the following suffixes [{}] OR [{}]: {}", 
+								ei.getSignature(), 
+								config.getConfig(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO), 
+								config.getConfig(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO), 
+								att);
 						continue;
 					}
 
@@ -377,6 +383,7 @@ public boolean downloadAttachmentForKaltura(EquellaItem ei) {
 							KEY_ATT_LINKS_VIEW);
 					bestFitAttFilesize = size;
 					bestFitAttFilename = filename;
+					bestFitFileSuffix = validSuffix;
 				} catch (Exception e) {
 					LOGGER.info("{} - Unable to check attachment - {}", ei.getSignature(), e.getMessage());
 				}
@@ -398,6 +405,7 @@ public boolean downloadAttachmentForKaltura(EquellaItem ei) {
 			if (FileUtils.downloadWithProgress(targetFile, bestFitAttLink, accessToken, Integer.parseInt(config.getConfig(Config.GENERAL_DOWNLOAD_CHATTER)),
 					bestFitAttFilesize)) {
 				ei.setFilepath(targetFile.getAbsolutePath());
+				ei.setPrimaryFileType(bestFitFileSuffix);
 				return true;
 			}
 

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/utils/EquellaItem.java

@@ -32,6 +32,7 @@ public class EquellaItem {
 	private String kalturaTags;
 	private String createdDateStr;
 	private Date createdDate;
+	private String primaryFileType;
 
 	public String getKalturaMediaId() {
 		return kalturaMediaId;
@@ -102,5 +103,12 @@ public Date getCreatedDate() {
 	public void setCreatedDate(Date date) {
 		createdDate = date;
 	}
+	public void setPrimaryFileType(String suffix) {
+		primaryFileType = suffix;
+	}
+
+	public String getPrimaryFileType() {
+		return primaryFileType;
+	}
 
 }

openequella-toolbox/src/main/java/org/apereo/openequella/tools/toolbox/utils/FileUtils.java

@@ -32,6 +32,7 @@
 import org.apache.http.impl.client.HttpClients;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apereo.openequella.tools.toolbox.Config;
 
 public class FileUtils {
 	private static Logger LOGGER = LogManager.getLogger(FileUtils.class);
@@ -123,4 +124,31 @@ private static boolean removeFile(String filename) {
 		}
 		return false;
 	}
+
+
+	/**
+	 * Checks if the filename ends with a known suffix, and returns the suffix (in uppercase) 
+	 * 
+	 * Currently supports the Audio and Video suffix configs
+	 * 
+	 * @param filename
+	 * @return the suffix in uppercase, null if not a known suffix.
+	 * @throws Exception
+	 */
+	public static String extractSuffix(Config config, String filename) {
+		List<String> audioSuffixes = config.getConfigAsList(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO);
+		for(String validSuffix : audioSuffixes) {
+			if(filename.toUpperCase().endsWith(validSuffix.trim())) {
+				return validSuffix.trim();
+			}
+		}
+
+		List<String> videoSuffixes = config.getConfigAsList(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO);
+		for(String validSuffix : videoSuffixes) {
+			if(filename.toUpperCase().endsWith(validSuffix.trim())) {
+				return validSuffix.trim();
+			}
+		}	
+		return null;
+	}
 }

openequella-toolbox/src/test/java/org/apereo/openequella/tools/toolbox/ConfigTest.java

@@ -16,8 +16,12 @@
 
 package org.apereo.openequella.tools.toolbox;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.util.Properties;
+
 import org.junit.Test;
 
 public class ConfigTest {
@@ -42,4 +46,122 @@ public void testConfigDateFormatConfigFile() {
 		}
 
 	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixes() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, "");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, "");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertTrue("Config is expected to be valid, but is not.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesReqVideo() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, "");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertFalse("Config is expected to be invalid, but was valid.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesReqAudio() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, "");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertFalse("Config is expected to be invalid, but was valid.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesUniqueNoAudio() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, "");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, ".mp4");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertTrue("Config is expected to be valid, but is not.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesUniqueNoVideo() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, ".mp3");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, "");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertTrue("Config is expected to be valid, but is not.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesUniqueFailedSingle() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, ".mp4");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, ".mp4");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertFalse("Config is expected to be invalid, but is valid.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesUniqueFailedMultiple() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, ".wma,.mp4");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, " .mp4, .mov");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertFalse("Config is expected to be invalid, but is valid.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
+
+	@Test
+	public void testConfigMigrateToKalturaAttachmentSuffixesUniqueGeneral() {
+		Properties props = new Properties();
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_AUDIO, ".mp3, .wma, .wav");
+		props.put(Config.OEQ_SEARCH_ATT_SUFFIXES_VIDEO, ".mp4,.mov");
+		try {
+			Config c = new Config(props);
+			c.checkMigrateToKalturaAttachmentSuffixes();
+			assertTrue("Config is expected to be valid, but is not.", c.isValidConfig());
+		} catch (Exception e) {
+			fail(e.getMessage());
+		}
+
+	}
 }

openequella-toolbox/src/test/java/org/apereo/openequella/tools/toolbox/ExportItemsDriverTest.java

@@ -19,7 +19,6 @@
 import static org.junit.Assert.assertEquals;
 
 import java.util.Properties;
-import java.util.UUID;
 
 import org.apereo.openequella.tools.toolbox.utils.EquellaItem;
 import org.json.JSONArray;