Merge branch 'master' of github.com:bcgov/wps-research

py/csv_simple_rasterize_onto.py

@@ -1,4 +1,5 @@
-'''20211216: burn point locations from a csv into a raster mask..
+'''DEPRECATED
+20211216: burn point locations from a csv into a raster mask..
 ..use this to convert csv groundref locations into format used by TRU students! 
 
 * input csv file must include columns named: row, col

py/misc.py

@@ -334,7 +334,7 @@ def xy_to_pix_lin(fn, x, y, nb):  # raster fn, lat/lon, number of bands (assume
            str(x), # lat
            str(y)] # long
     cmd = ' '.join(cmd)
-    #print(cmd)
+    print(cmd)
     lines = [x.strip() for x in os.popen(cmd).readlines()]
     count = 0
     if len(lines) >= 2 * (1 + nb):
@@ -360,7 +360,10 @@ def xy_to_pix_lin(fn, x, y, nb):  # raster fn, lat/lon, number of bands (assume
         row, col = lin_i, pix_i
         return row, col, data  # return the goods!
     else:
-        err("misc.py: unexpected output from gdallocationinfo: number of lines: " + str(len(lines)))
+        for line in lines:
+            print([line])
+        print("misc.py: unexpected output from gdallocationinfo: number of lines: " + str(len(lines)))
+        return None
 
 def pix_lin_to_xy(fn, col, row):
     err('fix this with code from raster_pixels_location.py')

py/raster_csv_latlon_extract_spectra.py

@@ -49,22 +49,44 @@
 lat_i, lon_i, name_i = -1, -1, -1
 for i in range(len(fields)):
     f = fields[i]
-    if len(f.split('lat')) > 1:
-        if lat_i != -1: err("more than one field matched lat")
-        else: lat_i = i
-    if len(f.split('lon')) > 1:
-        if lon_i != -1: err("more than one field matched lon")
-        else: lon_i = i    
+    if f.strip() == 'y':
+        if lat_i != -1:
+            err("more than one field matched lat")
+        else:
+            lat_i = i
+    if f.strip() == 'x':
+        if lon_i != -1:
+            err("more than one field matched lon")
+        else:
+            lon_i = i    
     if len(f.split('name')) > 1:
-        if name_i != -1: err("more than one field matched name")
-        else: name_i = i
+        if name_i != -1:
+            err("more than one field matched name")
+        else:
+            name_i = i
+
+# X,Y preferred but lat, lon accepted as well
+for i in range(len(fields)):
+    f = fields[i]
+    if len(f.split('lat')) > 1:
+        if lat_i == -1:
+            lat_i = i
+    if len(f.split('lon')) > 1 or len(f.split('X')) > 1:
+        if lon_i == -1:
+            lon_i = i
 
+print("LAT_I", lat_i)
+print("LON_I", lon_i)
 f = open(tgt_f, "wb")    
 f.write("feature_id,row,lin,xoff,yoff".encode())
 for line in data:
+    print("data line:", [line])
     lat, lon = line[lat_i], line[lon_i]
 
-    row, col, dat = xy_to_pix_lin(fn, lon, lat, int(nband))
+    result = xy_to_pix_lin(fn, lon, lat, int(nband))
+    if result is None:
+        continue 
+    row, col, dat = result
 
     print(','.join([str(x) for x in (line + [lat, lon, row, col, fn] + dat)]))
 

py/sentinel2_extract.py

@@ -1,11 +1,13 @@
 '''Can use this method to open Level1 format Sentinel-2 data downloaded either from:
-* ESA Copernicus, or from
-* Google Cloud Platform
+    *** ESA Copernicus ***,
+     or from
+    *** Google Cloud Platform ***
 
 Two cases:
 * after running gcp/fix_s2.py and zipping the folder
 * if there is no zip file (having downloaded from GCP
 
+NOTE: this script extracts products with different resolutions separately. They are NOT resampled into a common stack! 
 NOTE: no_stomp parameter avoids unzipping the zip file (again)'''
 import time
 from misc import *

py/sentinel2_stack_all.py

@@ -1,10 +1,8 @@
 '''20211121 same as sentinel2_extract_stack_all.py EXCEPT:
     this one doesn't extract from zip!!!!!!!! 
 
-WHY?????????????
-
-Assumption:
-    we've created L2 data with sen2cor
+WHY????????????? Assumption:
+ *** we've created L2 data with sen2cor, most likely after fetching data from py/gcp/update_tile.py
 
 A) extract Sentinel2, B) resample to 10m c) prefix bandnames with dates..
    D) stack everything!

py/sentinel2_stack_dates.py

@@ -0,0 +1,17 @@
+'''20240723 stack sentinel2 .bin files in temporal order
+
+one tile only supported'''
+from misc import run
+import os
+
+lines = [x.strip() for x in os.popen('ls -1 S*.bin').readlines()]
+
+# process in order
+lines = [[line.split('_')[2], line] for line in lines]
+lines.sort()
+lines = [line[1] for line in lines]
+
+for line in lines:
+    print(line)
+
+run(' '.join(['raster_stack.py'] + lines + ['stack.bin']))

py/setup.py

@@ -1,13 +1,14 @@
-from misc import *
+from misc import exists, run
+import os
 
 # setup for python stuff
 if not exists('../bin'):
     run('mkdir -p ../bin')
 
-run('sudo apt install python3-pip python3-setuptools')
-run('sudo apt install python3-gdal libgdal-dev gdal-bin python3-rasterio rasterio') #  simplekml')
 run('sudo apt update && sudo apt upgrade')
-run('python3 -m pip install numpy scikit-learn matplotlib alphashape descartes utm pyproj geopy')
+run('sudo apt install python3-pip python3-setuptools')
+run('sudo apt install python3-gdal libgdal-dev gdal-bin python3-rasterio rasterio awscli') #  simplekml')
+run('python3 -m pip install joblib numpy scikit-learn matplotlib alphashape descartes utm pyproj geopy')
 
 if not exists('../deb'):
     run('mkdir -p ../deb')
@@ -21,7 +22,8 @@
     bf = '../bin/' + stem # + '.exe'
     cmd = "g++ -w -O4 -march=native -o " + bf + " " + c + " ../cpp/misc.cpp -lpthread"  # -g 
     if stem != "misc" and not exists(bf):
-        run(cmd)
+        print(cmd)
+        a = os.system(cmd)
 
 py = os.popen("find ./*.py").readlines()
 for p in py:
@@ -46,4 +48,5 @@
     if w != "__init__.py" and not exists(bf):
         open(wf, 'wb').write('\n'.join(lines).encode())
         cmd = 'g++ -w -O3 -o ' + bf + ' ' + wf
-        run(cmd + "; rm -rf " + wf)
+        print(cmd)
+        a = os.system(cmd + "; rm -rf " + wf)