add stdout, add mergeSchema option to the read schema statement (#93)

* add stdout, add mergeSchema option to the read schema statement

* bump to 0.14.3

* add stdout for offline

fink_mm/__init__.py

@@ -12,6 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "0.14.2"
+__version__ = "0.14.3"
 __distribution_schema_version__ = "1.3"
 __observatory_schema_version__ = "1.1"

fink_mm/offline/spark_offline.py

@@ -413,10 +413,8 @@ def launch_offline_mode(arguments):
     stdout, stderr = process.communicate()
     if process.returncode != 0:  # pragma: no cover
         logger.error(
-            "fink-mm offline crossmatch application has ended with a non-zero returncode.\
-                \n\t cause:\n\t\t{}".format(
-                stderr
-            )
+            f"fink-mm offline crossmatch application has ended with a non-zero returncode.\
+                \n\tstdout: \n\n{stdout} \n\tstderr:\n\n{stderr}"
         )
         exit(1)
 

fink_mm/online/ztf_join_gcn.py

@@ -241,7 +241,7 @@ def ztf_join_gcn_stream(
     # )
 
     # Create a DF from the database
-    userschema = spark.read.parquet(gcn_rawdatapath).schema
+    userschema = spark.read.option('mergeSchema', True).parquet(gcn_rawdatapath).schema
 
     df_grb_stream = (
         spark.readStream.format("parquet")
@@ -453,10 +453,8 @@ def launch_joining_stream(arguments):
     stdout, stderr = process.communicate()
     if process.returncode != 0:  # pragma: no cover
         logger.error(
-            "fink-mm joining stream spark application has ended with a non-zero returncode.\
-                \n\t cause:\n\t\t{}".format(
-                stderr
-            )
+            f"fink-mm joining stream spark application has ended with a non-zero returncode.\
+                \n\tstdout:\n\n{stdout} \n\tstderr:\n\n{stderr}"
         )
         exit(1)