[yugabyte#1886] Adapt release code to work with the new tar.gz format…

… without -ce/-ee

Summary:
Some of the devops related code for release formatting was still relying on the -ce/-ee
components.

Ensured the YW code still handles -ee packages (for backwards compatibility with pre-existing releases), as well as new packages without the -ee.

Also made all the `version.txt` files be symlinks:
```
ls -la {,managed/,managed/devops/}version.txt
managed/devops/version.txt -> ../../version.txt
managed/version.txt -> src/main/resources/version.txt
version.txt -> managed/version.txt
```

The source of truth for `version.txt` lives in `managed/src/main/resources/version.txt`, due to YW build requirements.

Test Plan:
Adapted our integration tests to run over this.

This depends on D6918.

Reviewers: ram, mikhail

Reviewed By: mikhail

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D6944

managed/devops/opscli/ybops/utils/__init__.py

@@ -96,39 +96,37 @@ def extract_components_from_package_name(self, package_name, is_official_release
           eg: <repo>-<A.B.C.D>-b<build_number>-<system>-<machine>.tar.gz
         """
         # Always expect <repo>-<version>.
-        pattern = "^([^-]+)-({})".format(RELEASE_VERSION_PATTERN)
+        pattern = "^(?P<repo>[^-]+)-(?P<version>{})".format(RELEASE_VERSION_PATTERN)
         # If this is an official release, we expect a commit hash and maybe a build_type, else we
         # expect a "-b" and a build number.
         if is_official_release:
             # Add build number.
-            pattern += "-b([0-9]+)"
+            pattern += "-b(?P<build_number>[0-9]+)"
         else:
             # Add commit hash and maybe build type.
-            pattern += "-([^-]+)(-([^-]+))?"
-        pattern += "-([^-]+)-([^-]+)\.tar\.gz$"
+            pattern += "-(?P<commit_hash>[^-]+)-(?P<build_type>([^-]+))?"
+        pattern += "-(?P<system>[^-]+)-(?P<machine>[^-]+)\.tar\.gz$"
         match = re.match(pattern, package_name)
         if not match:
             raise YBOpsRuntimeError("Invalid package name format: {}".format(package_name))
-        self.repo, self.version, commit_or_build_number = match.group(1, 2, 3)
-        if is_official_release:
-            self.build_number = commit_or_build_number
-            self.commit = None
-        else:
-            self.build_number = None
-            self.commit = commit_or_build_number
-        if is_official_release:
-            self.system, self.machine = match.group(4, 5)
-        else:
-            # build_type should be None except for yugabyte. We ignore match 4 as -release
-            self.build_type, self.system, self.machine = match.group(5, 6, 7)
+        self.repo = match.group("repo")
+        self.version = match.group("version")
+        self.build_number = match.group("build_number") if is_official_release else None
+        self.commit = match.group("commit_hash") if not is_official_release else None
+        self.build_type = match.group("build_type") if not is_official_release else None
+        self.system = match.group("system")
+        self.machine = match.group("machine")
 
     def validate(self):
         if self.repo not in RELEASE_REPOS:
             raise YBOpsRuntimeError("Invalid repo {}".format(self.repo))
 
     def get_release_package_name(self):
-        return "{}-{}-{}-{}.tar.gz".format(
-            self.repo, self.get_release_name(), self.system, self.machine)
+        return "{repo}-{release_name}-{system}-{machine}.tar.gz".format(
+            repo=self.repo,
+            release_name=self.get_release_name(),
+            system=self.system,
+            machine=self.machine)
 
     def get_release_name(self):
         # If we have a build number set, prioritize that to get the release version name, rather

managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java

@@ -74,7 +74,10 @@ public static ReleaseMetadata create(String version) {
       FileSystems.getDefault().getPathMatcher("glob:**yugabyte*.tar.gz");
   Predicate<Path> packagesFilter = p -> Files.isRegularFile(p) && ybPackageMatcher.matches(p);
 
-  final Pattern ybPackagePattern = Pattern.compile("[^.]+yugabyte-ee-(.*)-centos(.*).tar.gz");
+  // This regex needs to support old style packages with -ee as well as new style packages without.
+  // There are previously existing YW deployments that will have the old packages and users will
+  // need to still be able to use said universes and their existing YB releases.
+  final Pattern ybPackagePattern = Pattern.compile("[^.]+yugabyte-(?:ee-)?(.*)-centos(.*).tar.gz");
 
   public Map<String, String> getLocalReleases(String localReleasePath) {
     Map<String, String> releaseMap = new HashMap<>();

managed/src/main/resources/version.txt

@@ -1 +1 @@
-1.3.0.0-b0
+1.3.1.0-b0

managed/src/test/java/com/yugabyte/yw/common/ReleaseManagerTest.java

@@ -63,15 +63,24 @@ public void tearDown() throws IOException {
   }
 
   private void createDummyReleases(List<String> versions, boolean multipleRepos, boolean inDockerPath) {
+    createDummyReleases(versions, multipleRepos, inDockerPath, true);
+  }
+
+  private void createDummyReleases(
+      List<String> versions, boolean multipleRepos, boolean inDockerPath,
+      boolean hasEnterpriseStr) {
     versions.forEach((version) -> {
       String versionPath = String.format("%s/%s", TMP_STORAGE_PATH, version);
       new File(versionPath).mkdirs();
       if (inDockerPath) {
         versionPath = TMP_DOCKER_STORAGE_PATH;
       }
-      createTempFile(versionPath, "yugabyte-ee-" + version + "-centos-x86_64.tar.gz", "Sample data");
+      String eeStr = hasEnterpriseStr ? "ee-" : "";
+      createTempFile(
+        versionPath, "yugabyte-" + eeStr + version + "-centos-x86_64.tar.gz", "Sample data");
       if (multipleRepos) {
-        createTempFile(versionPath, "devops.xyz." + version + "-centos-x86_64.tar.gz", "Sample data");
+        createTempFile(
+          versionPath, "devops.xyz." + version + "-centos-x86_64.tar.gz", "Sample data");
       }
     });
   }
@@ -149,6 +158,8 @@ public void testLoadReleasesWithReleaseAndDockerPath() {
     createDummyReleases(versions, false, false);
     List<String> dockerVersions = ImmutableList.of("0.0.2-b2");
     createDummyReleases(dockerVersions, false, true);
+    List<String> dockerVersionsWithoutEe = ImmutableList.of("0.0.3-b3");
+    createDummyReleases(dockerVersionsWithoutEe, false, true, false);
     releaseManager.importLocalReleases();
     ArgumentCaptor<ConfigHelper.ConfigType> configType;
     ArgumentCaptor<HashMap> releaseMap;
@@ -157,7 +168,9 @@ public void testLoadReleasesWithReleaseAndDockerPath() {
     Mockito.verify(configHelper, times(1)).loadConfigToDB(configType.capture(), releaseMap.capture());
     Map expectedMap = ImmutableMap.of(
         "0.0.1", TMP_STORAGE_PATH + "/0.0.1/yugabyte-ee-0.0.1-centos-x86_64.tar.gz",
-        "0.0.2-b2", TMP_STORAGE_PATH + "/0.0.2-b2/yugabyte-ee-0.0.2-b2-centos-x86_64.tar.gz");
+        "0.0.2-b2", TMP_STORAGE_PATH + "/0.0.2-b2/yugabyte-ee-0.0.2-b2-centos-x86_64.tar.gz",
+        "0.0.3-b3", TMP_STORAGE_PATH + "/0.0.3-b3/yugabyte-0.0.3-b3-centos-x86_64.tar.gz"
+    );
 
     assertEquals(SoftwareReleases, configType.getValue());
     assertReleases(expectedMap, releaseMap.getValue());

managed/ui/package.json

@@ -6,7 +6,7 @@
     "start": "react-app-rewired start",
     "start:prod": "pushstate-server build",
     "cleanup": "rimraf build && rimraf ../src/main/public",
-    "fetch-map": "aws s3 sync s3://no-such-url build/static/map",
+    "fetch-map": "aws s3 sync s3://$YB_MAP_URL build/static/map",
     "build": "npm run cleanup && react-app-rewired build && npm run fetch-map",
     "build-and-copy": "npm run build && ncp build ../src/main/public",
     "eject": "react-app-rewired eject",

managed/yb_release

@@ -4,7 +4,7 @@ set -euo pipefail
 
 # TODO: move this to common file.
 if [[ -z ${DEVOPS_HOME:-} ]]; then
-  devops_home_candidate_dir="${BASH_SOURCE%/*}"/../devops
+  devops_home_candidate_dir="${BASH_SOURCE%/*}"/devops
   if [[ -d $devops_home_candidate_dir ]]; then
     export DEVOPS_HOME=$( cd "$devops_home_candidate_dir" && pwd )
   else

version.txt

@@ -0,0 +1 @@
+managed/version.txt