-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathSSURGO_BatchDownload.py
1352 lines (1078 loc) · 56.1 KB
/
SSURGO_BatchDownload.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# ---------------------------------------------------------------------------
# SSURGO_BatchDownload.py
# Created on: 10-31-2013
# Author: Steve.Peaslee
# GIS Specialist
# National Soil Survey Center
# USDA - NRCS
# e-mail: [email protected]
# phone: 608.662.4422 ext. 216
# Author: Adolfo.Diaz
# GIS Specialist
# National Soil Survey Center
# USDA - NRCS
# e-mail: [email protected]
# phone: 608.662.4422 ext. 216
# ==========================================================================================
# Download SSURGO data from Web Soil Survey
# Uses Soil Data Access query to generate choicelist and URLs for each survey
#
# Three different tools call this script. One tool uses an Areasymbol wildcard to
# select surveys for download. Another tool uses an Areaname wildcard to
# elect surveys for download. The third uses an SAPOLYGON layer to generate a list
# of Areasymbol values to select surveys for download.
#
# Requires MS Access to run optional text file import for a custom SSURGO Template DB,
# as well as a modification to the VBA in the Template DB. Name of macro is BatchImport
# There are a lot of problems with WSS 3.0. One issue is trying to determine which surveys have
# spatial data. Normally this should be sapubstatuscode = 2.
# According to Gary, there is a finer level of detail available in the sastatusmap table.
# The columns tabularmudist and spatialmudist tell what kind of mapunit data is present in either the
# tabular or spatial portions. The possible values are:
#
# 1 = has ordinary mapunits and no NOTCOM mapunits
# 2 = has both ordinary and NOTCOM mapunits
# 3 = has only NOTCOM mapunits
# 4 = has no mapunits at all
# ==========================================================================================
#
# 10-31-2013
# 11-22-2013
# 01-08-2014
# 01-16-2014 Bad bug, downloads and unzips extra copy of some downloads. fixed.
# 01-22-2014 Modified interface to require that one of the batchimport mdb files be used.
# Posted all available state template databases to NRCS-GIS sharepoint
#
# Looking at potential for getting old downloads from the Staging Server. Lots of issues to consider...
# Staging Server URL requires E-Auth and has subdirectories
# 04-16-2014 https://soils-staging.sc.egov.usda.gov/NASIS_Export/Staging2Ssurgo/
#
# 05-13-2014 Modified unzip routine to handle other subfolder names at version 3.1 of WSS.
#
# 08-07-2014 Added function to find MS Access application by searching the Registry
# Looks under HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths
#
# 2014-09-27 Added post-import check to make sure tabular import was at least partially successful.
# Bails out if the SACATALOG table does not contain the SAVEREST date
#
# New version of script. Attempting to move most of the main code to functions so
# that failover works better. Breaks are a little messy and it wants to keep running no matter what.
#
# 2014-10-05 Added option to include NOTCOM survey using the tool validation query
#
# 2014-10-10 Removed requirement for MS Access. Now uses csv reader if the user chooses to perform
# the tabular import.
# 2014-10-13 Modified to populate the "SYSTEM - Mapunit Sort Specifications" table
# NEED TO DO THE SAME FOR THE "SYSTEM - INTERP DEPTH SEQUENCE TABLE"
# NEED TO LOOK AT IL177 legend.txt. Adolfo says this one will fail to import unless
# the csv reader is bumped up using csv.field_size_limit(sys.maxsize). Has failed at 128KB. Bumped to 512KB.
# Might also look at c = csv.reader(f, delimiter='|', quoting=csv.QUOTE_NONE)
# 2014-10-18 Modified SYSTEM table to only include cointerp records with ruledepth=0
# 2014-10-28 Increased sleep time before and after compact because of of errors
# 2014-10-30 Some problems adding MUNAME field to shapefile when output folder is on network share.
#
# 2015-03-13 Some improvements to the date-check/overwrite logic and messaging
#
# 2015-10-20 Added MUNAME, FARMLNDCL as an option
# 2015-10-20 Changed tabular import to truncate any values that exceed the field length (MUNAME Problem)
# ID604, ID670, WA651
# ==========================================================================================
# Updated 12/16/2016 - Adolfo Diaz
# Converted the SOAP request to POST-REST request to SDaccess. A.D.
# ==========================================================================================
# Updated 3/15/2021 - Adolfo Diaz
#
# - Updated and Tested for ArcGIS Pro 2.5.2 and python 3.6
# - All describe functions use the arcpy.da.Describe functionality.
# - All intermediate datasets are written to "in_memory" instead of written to a FGDB and
# and later deleted. This avoids having to check and delete intermediate data during every
# execution.
# - All cursors were updated to arcpy.da
# - Added code to remove layers from an .aprx rather than simply deleting them
# - Updated AddMsgAndPrint to remove ArcGIS 10 boolean and gp function
# - Updated errorMsg() Traceback functions slightly changed for Python 3.6.
# - Added parallel processing factor environment
# - swithced from sys.exit() to exit()
# - All gp functions were translated to arcpy
# - Every function including main is in a try/except clause
# - Main code is wrapped in if __name__ == '__main__': even though script will never be
# used as independent library.
# - Normal messages are no longer Warnings unnecessarily.
# ===============================================================================================================
def AddMsgAndPrint(msg, severity=0):
# prints message to screen if run as a python script
# Adds tool message to the geoprocessor
#
#Split the message on \n first, so that if it's multiple lines, a GPMessage will be added for each line
try:
print(msg)
#for string in msg.split('\n'):
#Add a geoprocessing message (in case this is run as a tool)
if severity == 0:
arcpy.AddMessage(msg)
elif severity == 1:
arcpy.AddWarning(msg)
elif severity == 2:
arcpy.AddError("\n" + msg)
except:
pass
# ================================================================================================================
def errorMsg():
try:
exc_type, exc_value, exc_traceback = sys.exc_info()
theMsg = "\t" + traceback.format_exception(exc_type, exc_value, exc_traceback)[1] + "\n\t" + traceback.format_exception(exc_type, exc_value, exc_traceback)[-1]
if theMsg.find("exit") > -1:
AddMsgAndPrint("\n\n")
pass
else:
AddMsgAndPrint(theMsg,2)
except:
AddMsgAndPrint("Unhandled error in unHandledException method", 2)
pass
## ===================================================================================
def Number_Format(num, places=0, bCommas=True):
try:
# Format a number according to locality and given places
#locale.setlocale(locale.LC_ALL, "")
if bCommas:
theNumber = locale.format("%.*f", (places, num), True)
else:
theNumber = locale.format("%.*f", (places, num), False)
return theNumber
except:
errorMsg()
return ""
## ===================================================================================
def CheckMSAccess():
# Not using this function any more
#
# Make sure this computer has MS Access installed so that the tabular import will run
try:
msa = "MSACCESS.EXE"
aReg = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
aKey = OpenKey(aReg, r"SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths")
acccessPath = ""
for i in range(1024):
keyName = EnumKey(aKey, i)
if keyName == msa:
subKey = OpenKey(aKey, keyName)
installPath = QueryValueEx(subKey, "Path")
accessPath = os.path.join(installPath[0], msa)
break
return accessPath
except WindowsError:
return ""
except:
errorMsg()
return ""
## ===================================================================================
def GetPublicationDate(areaSym):
#
#
#
# Please Note!!! Funtion not being used at this time
# Alternate method of getting SSURGO publication date using SDM Access query
#
# This should use SASTATUSMAP table instead of SACATALOG
# Add 'AND SAPUBSTATUSCODE = 2'
#
# Test version of SDA: http://sdmdataaccessha.dev.sc.egov.usda.gov/
#
import time, datetime
import xml.etree.cElementTree as ET
try:
# date formatting
# today = datetime.date.today()
# myDate = today + datetime.timedelta(days = -(self.params[0].value))
# myDate = str(myDate).replace("-","")
# wc = "'" + self.params[1].value + "%' AND SAVEREST > '" + myDate + "'"
# return list sorted by date
#SELECT S.AREASYMBOL, CONVERT (varchar(10), [SAVEREST], 126) AS SDATE FROM SACATALOG S WHERE AREASYMBOL LIKE 'KS%'
#sQuery = "SELECT CONVERT(varchar(10), [SAVEREST], 126) AS SAVEREST FROM SACATALOG WHERE AREASYMBOL = '" + areaSym + "'"
sQuery = "SELECT CONVERT(varchar(10), [SAVEREST], 126) AS SAVEREST FROM SASTATUSMAP WHERE AREASYMBOL = '" + areaSym + "' AND SAPUBSTATUSCODE = 2"
# Create request using JSON, return data as JSON
dRequest = dict()
dRequest["format"] = "JSON"
dRequest["query"] = sQuery
jData = json.dumps(dRequest) # {"QUERY": "SELECT AREASYMBOL, AREANAME, CONVERT(varchar(10), [SAVEREST], 126) AS SAVEREST FROM SASTATUSMAP WHERE AREASYMBOL LIKE \'WI025\' ORDER BY AREASYMBOL", "FORMAT": "JSON"}
# Send request to SDA Tabular service using urllib2 library
# ArcPro Request
jData = jData.encode('ascii')
response = urllib.request.urlopen(url,jData)
jsonString = response.read() # {"Table":[["WI025","Dane County, Wisconsin","2016-09-27"]]}
# Convert the returned JSON string into a Python dictionary.
data = json.loads(jsonString) # {u'Table': [[u'WI025', u'Dane County, Wisconsin', u'2016-09-27']]}
return data['Table'][0][0]
""" ------------------------------------------- This is the original SOAP request; being replaced by POST-REST request --------------------------------------"""
## # Send XML query to SDM Access service
## #
## sXML = """<?xml version="1.0" encoding="utf-8"?>
## <soap12:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap12="http://www.w3.org/2003/05/soap-envelope">
## <soap12:Body>
## <RunQuery xmlns="http://SDMDataAccess.nrcs.usda.gov/Tabular/SDMTabularService.asmx">
## <Query>""" + sQuery + """</Query>
## </RunQuery>
## </soap12:Body>
## </soap12:Envelope>""" # Original version SDA
##
## dHeaders = dict()
## dHeaders["Host"] = "sdmdataaccess.nrcs.usda.gov" # Original SDA version
## dHeaders["Content-Type"] = "text/xml; charset=utf-8"
## dHeaders["SOAPAction"] = "http://SDMDataAccess.nrcs.usda.gov/Tabular/SDMTabularService.asmx/RunQuery" # Original version SDA" # Test version SDA
## dHeaders["Content-Length"] = len(sXML)
## sURL = "SDMDataAccess.nrcs.usda.gov" # original SDA
##
## # Create SDM connection to service using HTTP
## conn = httplib.HTTPConnection(sURL, 80)
##
## # Send request in XML-Soap
## conn.request("POST", "/Tabular/SDMTabularService.asmx", sXML, dHeaders)
##
## # Get back XML response
## response = conn.getresponse()
## xmlString = response.read()
##
## # Close connection to SDM
## conn.close()
##
## # Convert XML to tree format
## tree = ET.fromstring(xmlString)
##
## iCnt = 0
## # Create empty value list
## valList = list()
##
## # Iterate through XML tree, finding required elements...
## for rec in tree.iter():
##
## if rec.tag == "SAVEREST":
## # get the YYYYMMDD part of the datetime string
## # then reformat to match SQL query
## sdmDate = str(rec.text).split(" ")[0]
except:
errorMsg()
return 0
## ===================================================================================
def SSURGOVersion(newDB, tabularFolder):
# Get SSURGO version from the Template database "SYSTEM Template Database Information" table
#
# Ideally we want to compare with the value in version.txt with the version in
# the "SYSTEM - Template Database Information" table. If they are not the same
# the tabular import should be aborted. There are some more specifics about the
# SSURGO version.txt valu in one of the Import macros of the Template database.
# Need to follow up and research this more.
# At this time we are only checking the first 'digit' of the string value.
#
# Should be able to get this to work using wildcard for fields and then
# use the version.txt as an alternative or failover.
try:
# Valid SSURGO version for data model. Ensures
# compatibility between template database and SSURGO download.
versionTxt = os.path.join(tabularFolder, "version.txt")
if not arcpy.Exists(newDB):
AddMsgAndPrint("Missing input database (" + newDB + ")",2)
return False
if arcpy.Exists(versionTxt):
# read just the first line of the version.txt file
fh = open(versionTxt, "r")
txtVersion = fh.readline().split(".")[0]
fh.close()
else:
# Unable to compare vesions. Warn user but continue
AddMsgAndPrint("Unable to find file: version.txt", 1)
return True
systemInfo = os.path.join(newDB, "SYSTEM - Template Database Information")
if arcpy.Exists(systemInfo):
# Get SSURGO Version from template database
dbVersion = 0
with arcpy.da.SearchCursor(systemInfo, "*", "") as srcCursor:
for rec in srcCursor:
if rec[0] == "SSURGO Version":
dbVersion = str(rec[2]).split(".")[0]
#AddMsgAndPrint("\tSSURGO Version from DB: " + dbVersion, 1)
del systemInfo
del newDB
if txtVersion != dbVersion:
# SSURGO Versions do not match. Warn user but continue
AddMsgAndPrint("Discrepancy in SSURGO Version number for Template database and SSURGO download", 1)
else:
# Unable to open SYSTEM table in existing dataset
# Warn user but continue
AddMsgAndPrint("Unable to open 'SYSTEM - Template Database Information'", 1)
return True
except:
errorMsg()
return False
## ===================================================================================
def GetTemplateDate(newDB):
# Get SAVEREST date from previously existing Template database
# Use it to compare with the date from the WSS dataset
# If the existing database is same or newer, it will be kept and the WSS version skipped
#
# da cursor will return: datetime.datetime(2014, 12, 1, 15, 22, 8)
# Should be able to reformat to an integer value for comparison with filename-imbedded date.
#
try:
#if not arcpy.Exists(newDB): # Check for existance before calling this function
# return 0
saCatalog = os.path.join(newDB, "SACATALOG")
dbDate = 0
if arcpy.Exists(saCatalog):
dateObj = None
with arcpy.da.SearchCursor(saCatalog, ("SAVEREST"), "[AREASYMBOL] = '" + areaSym + "'") as srcCursor:
for rec in srcCursor:
# Reformat datetime to YYYYMMDD and convert to integer
#dateObj = int(rec[0].strftime('%Y%m%d'))
dateObj = rec[0]
if dateObj is None:
return 0
intDate = "%Y%m%d" # YYYYMMDD format for comparison
dbDate = int(dateObj.strftime(intDate))
else:
AddMsgAndPrint("SACATALOG table in Template database not found",2)
return 0
return dbDate
except:
errorMsg()
return 0
## ===================================================================================
def GetTabularDate(newFolder):
# Get string for SAVEREST date from tabular/sacatlog.txt file
# Use it to compare with the date from the WSS dataset
# If the existing database is same or newer, it will be kept and the WSS version skipped
# The original string looks like this: 12/05/2013 23:44:00
#
# Return YYYYMMDD as integer
try:
tabDate = 0
# Try finding the text file in the tabular folder and reading SAVEREST from that file.
saCatalog = os.path.join(newFolder, r"tabular\sacatlog.txt")
if arcpy.Exists(saCatalog):
fh = open(saCatalog, "r")
rec = fh.readline()
fh.close()
# Example date (which is index 3 in pipe-delimited file): 9/23/2014 6:49:27
vals = rec.split("|")
recDate = vals[3]
wssDate = "%m/%d/%Y %H:%M:%S" # string date format used for SAVEREST in text file
intDate = "%Y%m%d" # YYYYMMDD format for comparison
dateObj = datetime.strptime(recDate, wssDate)
tabDate = int(dateObj.strftime(intDate))
else:
AddMsgAndPrint(" \nUnable to find file: " + saCatalog, 1)
return tabDate
except:
errorMsg()
return tabDate
## ===================================================================================
def GetDownload(areasym, surveyDate, importDB):
# download survey from Web Soil Survey URL and return name of the zip file
# want to set this up so that download will retry several times in case of error
# return empty string in case of complete failure. Allow main to skip a failed
# survey, but keep a list of failures
#
# Only the version of zip file without a Template database is downloaded. The user
# must have a locale copy of the Template database that has been modified to allow
# automatic tabular imports.
# create URL string from survey string and WSS 3.0 cache URL
baseURL = "https://websoilsurvey.sc.egov.usda.gov/DSD/Download/Cache/SSA/"
try:
# List of states that use a Template database other than US_2003.
# This list will have to be updated in the future if it is used to
# get downloads with the Template database included in the zipfile.
dbInfo = {'AK':'AK', 'CT':'CT', 'FL':'FL', 'GA':'GA', 'HI':'HI', 'IA':'IA', \
'ID':'ID', 'IN':'IN', 'ME':'ME', 'MI':'MI', 'MN':'MN', 'MT':'MT', 'NC':'NC', \
'NE':'NE', 'NJ':'NJ', 'OH':'OH', 'OR':'OR', 'PA':'PA', 'SD':'SD', 'UT':'UT', \
'VT':'VT', 'WA':'WA', 'WI':'WI', 'WV':'WV', 'WY':'WY', 'FM':'HI', 'PB':'HI'}
# Incorporate the name of the Template database into the URL
st = areaSym[0:2]
if st in dbInfo:
db = "_soildb_" + dbInfo[st] + "_2003"
else:
db = "_soildb_US_2003"
# Use this zipfile for downloads without the Template database
zipDate = str(surveyDate)[0:4] + "-" + str(surveyDate)[4:6] + "-" + str(surveyDate)[6:8]
zipName = "wss_SSA_" + areaSym + "_[" + str(zipDate) + "].zip"
# Use this URL for downloads with the state or US_2003 database
#zipName = "wss_SSA_" + areaSym + db + "_[" + surveyDate + "].zip"
zipURL = baseURL + zipName
AddMsgAndPrint("\tDownloading survey " + areaSym + " from Web Soil Survey...", 0)
# Open request to Web Soil Survey for that zip file
request = urlopen(zipURL)
# set the download's output location and filename
local_zip = os.path.join(outputFolder, zipName)
# make sure the output zip file doesn't already exist
if os.path.isfile(local_zip):
os.remove(local_zip)
# save the download file to the specified folder
output = open(local_zip, "wb")
output.write(request.read())
output.close()
del request
del output
# if we get this far then the download succeeded
return zipName
except HTTPError as e:
AddMsgAndPrint('HTTP Error' + str(e),2)
return ""
except URLError as e:
AddMsgAndPrint('URL Error' + str(e),2)
return ""
except socket.timeout as e:
AddMsgAndPrint('Soil Data Access timeout error',2)
return ""
except socket.error as e:
AddMsgAndPrint('Socket error: ' + str(e),2)
return ""
except httplib.BadStatusLine:
AddMsgAndPrint("\t\t" + areasym + " - Web Soil Survey connection failure", 1)
return ""
except:
# problem deleting partial zip file after connection error?
# saw some locked, zero-byte zip files associated with connection errors
AddMsgAndPrint("\tFailed to download zipfile", 0)
errorMsg()
return ""
sleep(1)
return ""
## ===================================================================================
def CheckExistingDataset(areaSym, surveyDate, newFolder, newDB):
try:
bNewer = True # Default setting should result in overwriting the current data if it already exists
#AddMsgAndPrint(" \nChecking newFolder: " + newFolder, 1)
if os.path.isdir(newFolder):
# This survey appears to have already been downloaded. Check to see if it is complete.
# If not complete, overwrite it.
# Need to handle situations where Tabular data was not imported. Right now this will
# throw an error!
#bNewer = False # Default setting should result in overwriting the current data if it already exists
# Having a new issue with date comparisons. Do I need to re-order the date string
# to YYYYMMDD in order to compare the filename date and the SAVEREST dates as integer?
#
# Another issue. If the data was previously downloaded but the option to use a
# Template database was changed then it gets a little complicated. Should I bail
# if the specified database doesn't exist instead of failing over to looking at the
# date from the text file?
#
if newDB == "":
# No tabular import will be performed, use the text file to get the date
dbDate = GetTabularDate(newFolder)
elif os.path.isfile(newDB):
# Template database exists, get date from the SACATALOG table
dbDate = GetTemplateDate(newDB)
if dbDate == 0:
AddMsgAndPrint(" \nLocal dataset " + areaSym + " already exists but is incomplete", 1)
else:
AddMsgAndPrint(" \nLocal dataset for " + areaSym + " already exists (date of " + str(dbDate) + ")", 0)
else:
# Missing database even though a path was given by the user
AddMsgAndPrint("\tMissing database (" + newDB + ")", 1)
dbDate = 0
if dbDate == 0:
# Could not get SAVEREST date from database, assume old dataset is incomplete and overwrite
#AddMsgAndPrint("\tLocal dataset is incomplete and will be overwritten", 1)
shutil.rmtree(newFolder, True)
sleep(3)
bNewer = True
if arcpy.Exists(newFolder):
AddMsgAndPrint("Failed to delete old dataset (" + newFolder + ")",2)
return False
else:
# Compare SDM date with local database date
if surveyDate > dbDate:
# Downloaded data is newer than the local copy. Delete and replace with new data.
#
#AddMsgAndPrint("\tReplacing local dataset with newer download", 1)
bNewer = True
# delete old data folder
shutil.rmtree(newFolder, True)
sleep(3)
if arcpy.Exists(newFolder):
AddMsgAndPrint("Failed to delete old dataset (" + newFolder + ")",2)
return False
else:
# according to the filename-date, the WSS version is the same or older
# than the local Template DB, skip download for this survey
if surveyDate == dbDate:
AddMsgAndPrint(" \nSkipping survey " + areaSym + ", local version is already current", 1)
else:
AddMsgAndPrint(" \nSkipping survey " + areaSym + ", local version is newer (" + str(dbDate) + ") than the WSS data!?", 1)
bNewer = False
else:
# This is a new download
bNewer = True
return bNewer
except:
errorMsg()
return False
## ===================================================================================
def ProcessSurvey(outputFolder, importDB, areaSym, bImport, bRemoveTXT, iGet, iTotal):
# Download and import the specified SSURGO dataset
try:
survey = asDict[areaSym]
env.workspace = outputFolder
surveyInfo = survey.split(",")
areaSym = surveyInfo[0].strip().upper()
# get date string
surveyDate = int(surveyInfo[1].strip().replace("-", ""))
# get survey name
surveyName = surveyInfo[2].strip()
# set standard final path and name for template database
newFolder = os.path.join(outputFolder, "soil_" + areaSym.lower())
# set standard name and path for SSURGO Template database
# Should I set this variable even when no import has been specified? I
# think this is causing problems.
if bImport:
newDB = os.path.join(os.path.join(newFolder, "tabular"), "soil_d_" + areaSym.lower() + ".mdb")
else:
newDB = ""
# check to make sure this survey hasn't already been downloaded
# This database-check won't work if the user was not running the tabular import.
# Need to add the option to look at the tabular text file to get the SAVEREST date
# when bImport is False
#
bNewer = CheckExistingDataset(areaSym, surveyDate, newFolder, newDB)
if bNewer:
# Get new SSURGO download or replace an older version of the same survey
# Otherwise skip download
#
AddMsgAndPrint(" \nProcessing survey " + areaSym + " (" + str(iGet) + " of " + str(iTotal) + "): " + surveyName, 0)
# First attempt to download zip file
zipName = GetDownload(areaSym, surveyDate, importDB)
if zipName == "" or zipName is None:
# Try downloading zip file a second time
sleep(5)
zipName = GetDownload(areaSym, surveyDate, importDB)
if zipName == "" or zipName is None:
# Failed second attempt to download zip file
# Give up on this survey
return "Failed"
bZip = UnzipDownload(outputFolder, newFolder, importDB, zipName)
if not bZip:
# Try unzipping a second time
sleep(1)
bZip = UnzipDownload(outputFolder, newFolder, importDB, zipName)
if not bZip:
# Failed second attempt to unzip
# Give up on this survey
return "Failed"
# Import tabular. Only try once.
if bImport:
if not ImportTabular(areaSym, newFolder, importDB, newDB, bRemoveTXT):
# Bail clear out of the whole download process
return "Failed"
return "Successful"
else:
# Existing local dataset is same age or newer than downloaded version
# skip it
return "Skipped"
except:
errorMsg()
return "Failed"
## ===================================================================================
def UnzipDownload(outputFolder, newFolder, importDB, zipName ):
# Given zip file name, try to unzip it
try:
local_zip = os.path.join(outputFolder, zipName)
if os.path.isfile(local_zip):
# got a zip file, go ahead and extract it
zipSize = (os.stat(local_zip).st_size / (1024.0 * 1024.0))
if zipSize > 0:
# Download appears to be successful
AddMsgAndPrint("\tUnzipping " + zipName + " (" + Number_Format(zipSize, 3, True) + " MB)...", 0)
with zipfile.ZipFile(local_zip, "r") as z:
# a bad zip file returns exception zipfile.BadZipFile
z.extractall(outputFolder)
# remove zip file after it has been extracted,
# allowing a little extra time for file lock to clear
sleep(3)
os.remove(local_zip)
# rename output folder to NRCS Geodata Standard for Soils
if os.path.isdir(os.path.join(outputFolder, zipName[:-4])):
# this is an older zip file that has the 'wss_' directory structure
os.rename(os.path.join(outputFolder, zipName[:-4]), newFolder)
elif os.path.isdir(os.path.join(outputFolder, areaSym.upper())):
# this must be a newer zip file using the uppercase AREASYMBOL directory
os.rename(os.path.join(outputFolder, areaSym.upper()), newFolder)
elif os.path.isdir(newFolder):
# this is a future zip file using the correct field office naming convention (soil_ne109)
# it does not require renaming.
pass
else:
# none of the subfolders within the zip file match any of the expected names
AddMsgAndPrint("Subfolder within the zip file does not match any of the standard names",2)
return False
else:
# Downloaded a zero-byte zip file
# download for this survey failed, may try again
AddMsgAndPrint("\tEmpty zip file downloaded for " + areaSym + ": " + surveyName, 1)
os.remove(local_zip)
return True
else:
# Don't have a zip file, need to find out circumstances and document
# rename downloaded database using standard convention, skip import
AddMsgAndPrint("Missing zip file (" + local_zip + ")",2)
return False
except zipfile.BadZipfile:
AddMsgAndPrint("Bad zip file?", 2)
return False
except:
errorMsg()
return False
## ===============================================================================================================
def GetTableInfo(newDB):
# Adolfo's function
#
# Retrieve physical and alias names from MDSTATTABS table and assigns them to a blank dictionary.
# Stores physical names (key) and aliases (value) in a Python dictionary i.e. {chasshto:'Horizon AASHTO,chaashto'}
# Fieldnames are Physical Name = AliasName,IEfilename
try:
tblInfo = dict()
# Open mdstattabs table containing information for other SSURGO tables
theMDTable = "mdstattabs"
env.workspace = newDB
# Establishes a cursor for searching through field rows. A search cursor can be used to retrieve rows.
# This method will return an enumeration object that will, in turn, hand out row objects
if arcpy.Exists(os.path.join(newDB, theMDTable)):
fldNames = ["tabphyname","tablabel","iefilename"]
with arcpy.da.SearchCursor(os.path.join(newDB, theMDTable), fldNames) as rows:
for row in rows:
# read each table record and assign 'tabphyname' and 'tablabel' to 2 variables
physicalName = row[0]
aliasName = row[1]
importFileName = row[2]
# i.e. {chaashto:'Horizon AASHTO',chaashto}; will create a one-to-many dictionary
# As long as the physical name doesn't exist in dict() add physical name
# as Key and alias as Value.
#if not physicalName in tblAliases:
if not importFileName in tblInfo:
#AddMsgAndPrint("\t" + importFileName + ": " + physicalName, 1)
tblInfo[importFileName] = physicalName, aliasName
del theMDTable
return tblInfo
else:
# The mdstattabs table was not found
AddMsgAndPrint("Missing mdstattabs table",2)
return tblInfo
except:
errorMsg()
return tblInfo
## ===================================================================================
def SortMapunits(newDB):
# Populate table 'SYSTEM - Mapunit Sort Specifications'. Required for Soil Data Viewer
# Looks like an alpha sort on AREASYMBOL, then MUSYM will work to set
# lseq and museq values within the "SYSTEM - Mapunit Sort Specifications" table
#
# Problem, this sort does not handle a mix of alpha and numeric musym values properly
#
# Populate table "SYSTEM - INTERP DEPTH SEQUENCE" from COINTERP using cointerpkey and seqnum
#
try:
# Make query table using MAPUNIT and LEGEND tables and use it to assemble all
# of the data elements required to create the "SYSTEM - Mapunit Sort Specification" table
inputTbls = ["legend", "mapunit"]
fldList = "legend.areasymbol areasymbol;legend.lkey lkey; mapunit.musym musym; mapunit.mukey mukey"
sqlJoin = "mapunit.lkey = legend.lkey"
queryTbl = "musorted"
# Cleanup
if arcpy.Exists(queryTbl):
arcpy.Delete_management(queryTbl)
# Find output SYSTEM table
sysFields = ["lseq", "museq", "lkey", "mukey"]
sysTbl = os.path.join(newDB, "SYSTEM - Mapunit Sort Specifications")
if not arcpy.Exists(sysTbl):
AddMsgAndPrint("Could not find " + sysTbl,2)
return False
arcpy.MakeQueryTable_management(inputTbls, queryTbl, "ADD_VIRTUAL_KEY_FIELD", "", fldList, sqlJoin)
# Open the query table, sorting on areasymbol
#sqlClause = [None, "order by legend_areasymbol asc"]
dMapunitSort = dict() # dictionary to contain list of musyms for each survey. Will be sorted
dMapunitData = dict() # dictionary for containing all neccessary data for SYSTEM -Map Unit Sort Specification
convert = lambda text: int(text) if text.isdigit() else text.lower()
alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key)]
with arcpy.da.SearchCursor(queryTbl, ["legend_areasymbol", "legend_lkey", "mapunit_musym", "mapunit_mukey"]) as cur:
for rec in cur:
areaSym = rec[0].encode('ascii')
lkey = rec[1].encode('ascii')
musym = rec[2].encode('ascii')
mukey = rec[3].encode('ascii')
# Append muysm values to dictionary by areasymbol key
if areaSym in dMapunitSort:
musymList = dMapunitSort[areaSym]
musymList.append(musym)
dMapunitSort[areaSym] = musymList
else:
dMapunitSort[areaSym] = [musym]
# store legend and map unit keys by areasymbol and map unit symbol
dMapunitData[(areaSym, musym)] = (lkey, mukey)
# Iterate through dMapunitSort dictionary, sorting muysm values
areaList = sorted(dMapunitSort.keys()) # sorted list of areasymbols
lseq = 0
mseq = 0
# Now read the dictionary back out in sorted order and populate the SYSTEM - Mapunit Sort Specifications table
#
with arcpy.da.InsertCursor(sysTbl, "*") as outCur:
for areaSym in areaList:
#AddMsgAndPrint(" \nProcessing survey: " + areaSym, 1)
lseq += 1
musymList = sorted(dMapunitSort[areaSym], key = alphanum_key)
for musym in musymList:
mseq += 1
mKey = (areaSym, musym)
lkey, mukey = dMapunitData[(areaSym, musym)]
outrec = lseq, mseq, lkey, mukey
outCur.insertRow(outrec)
# Populate "SYSTEM - INTERP DEPTH SEQUENCE" fields: cointerpkey and depthseq
# from COINTERP fields: cointerpkey and seqnum
# I am assuming that the cointerp table is already sorted. Is that safe??
#
#AddMsgAndPrint("\tUpdating SYSTEM - Interp Depth Sequence", 1)
inTbl = os.path.join(newDB, "cointerp")
inFlds = ["cointerpkey", "seqnum"]
outTbl = os.path.join(newDB, "SYSTEM - INTERP DEPTH SEQUENCE")
outFlds = ["cointerpkey", "depthseq"]
interpSQL = "ruledepth = 1"
with arcpy.da.SearchCursor(inTbl, inFlds, interpSQL) as sCur:
outCur = arcpy.da.InsertCursor(outTbl, outFlds)
for inRec in sCur:
outCur.insertRow(inRec)
return True
except:
errorMsg()
return False
## ===================================================================================
def ImportTabular(areaSym, newFolder, importDB, newDB, bRemoveTXT):
# Given zip file name, try to unzip it
#
# Problem with SACATALOG tabular for OR628 as of 12-12-2014
# Seemed to work OK for my laptop, but failed several times on workstation
# when output was to network share.
try:
# get database name from file listing in the new folder
env.workspace = newFolder
# move to tabular folder
env.workspace = os.path.join(newFolder, "tabular")
# copy over master database and run tabular import
AddMsgAndPrint("\tCopying selected master template database to tabular folder...", 0)
# copy user specified database to the new folder
shutil.copy2(importDB, newDB)
# Run Auto_Import routine which will import the tabular data from text files
AddMsgAndPrint("\tImporting textfiles into new database " + os.path.basename(newDB) + "...", 0)
# Using Adolfo's csv reader method to import tabular data from text files...
tabularFolder = os.path.join(newFolder, "tabular")
# if the tabular directory is empty return False
if len(os.listdir(tabularFolder)) < 1:
AddMsgAndPrint("No text files found in the tabular folder",2)
return False
if not SSURGOVersion(newDB, tabularFolder):
return False
# Create a dictionary with table information
tblInfo = GetTableInfo(newDB)
if len(tblInfo) == 0:
AddMsgAndPrint("Failed to get information from mdstattabs table",2)
return False
# Create a list of textfiles to be imported. The import process MUST follow the
# order in this list in order to maintain referential integrity. This list
# will need to be updated if the SSURGO data model is changed in the future.
#
txtFiles = ["distmd","legend","distimd","distlmd","lareao","ltext","mapunit", \
"comp","muaggatt","muareao","mucrpyd","mutext","chorizon","ccancov","ccrpyd", \
"cdfeat","cecoclas","ceplants","cerosnac","cfprod","cgeomord","chydcrit", \
"cinterp","cmonth", "cpmatgrp", "cpwndbrk","crstrcts","csfrags","ctxfmmin", \
"ctxmoicl","ctext","ctreestm","ctxfmoth","chaashto","chconsis","chdsuffx", \
"chfrags","chpores","chstrgrp","chtext","chtexgrp","chunifie","cfprodo","cpmat","csmoist", \
"cstemp","csmorgc","csmorhpp","csmormr","csmorss","chstr","chtextur", \
"chtexmod","sacatlog","sainterp","sdvalgorithm","sdvattribute","sdvfolder","sdvfolderattribute"]
# Need to add featdesc import as a separate item (ie. spatial\soilsf_t_al001.txt: featdesc)
# Static Metadata Table that records the metadata for all columns of all tables
# that make up the tabular data set.
mdstattabsTable = os.path.join(env.workspace, "mdstattabs")
# set progressor object which allows progress information to be passed for every merge complete
arcpy.SetProgressor("step", "Importing tabular data", 0, len(txtFiles) + 2, 1)
# Need to import text files in a specific order or the MS Access database will
# return an error due to table relationships and key violations
# Problem with length of some memo fields, need to allocate more memory
#csv.field_size_limit(sys.maxsize)
csv.field_size_limit(512000)
for txtFile in txtFiles:
# Get table name and alias from dictionary
if txtFile in tblInfo:
tbl, aliasName = tblInfo[txtFile]
else:
AddMsgAndPrint("Textfile reference '" + txtFile + "' not found in 'mdstattabs table'",2)
return False
arcpy.SetProgressorLabel("Importing " + tbl + "...")
# Full path to SSURGO text file
txtPath = os.path.join(tabularFolder, txtFile + ".txt")
# continue if the target table exists