[ZTP] Improvements to allow in-band zero touch provisioning

src/usr/lib/python3/dist-packages/ztp/Downloader.py

@@ -247,6 +247,10 @@ def getUrl(self, url=None, dst_file=None, incl_http_headers=None, is_secure=True
             else:
                 break
 
-        os.chmod(dst_file, stat.S_IRWXU)
+        try:
+            os.chmod(dst_file, stat.S_IRWXU)
+        except FileNotFoundError:
+            return (20, None)
+
         # Use curl result
         return (0, dst_file)

src/usr/lib/python3/dist-packages/ztp/ZTPSections.py

@@ -91,7 +91,7 @@ def __buildDefaults(self, section):
          @param section (dict) Configuration Section input data read from JSON file.
 
         '''
-        default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure']
+        default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure', 'pre-ztp-plugin-download']
         # Loop through objects and update them with default values
         for key in default_objs:
             _val = getField(section, key, bool, getCfg(key))

src/usr/lib/python3/dist-packages/ztp/defaults.py

@@ -48,6 +48,7 @@
   "log-file"             : "/var/log/ztp.log", \
   "log-level"            : "INFO", \
   "monitor-startup-config" : True, \
+  "pre-ztp-plugin-download" : True, \
   "restart-ztp-interval": 300, \
   "reboot-on-success"    : False, \
   "reboot-on-failure"    : False, \

src/usr/lib/ztp/ztp-engine.py

@@ -432,13 +432,54 @@ def __evalZTPResult(self):
         # Check reboot on result flags and take action
         self.__rebootAction(self.objztpJson.ztpDict, delayed_reboot=True)
 
+    def __downloadPlugins(self):
+        '''!
+         Check and download plugins used by configuration sections
+           @return  False - If failed to download one more plugins of a required configuration section
+                    True  - Successfully downloaded plugins of all configuration sections
+        '''
+
+        # Obtain a copy of the list of configuration sections
+        section_names = list(self.objztpJson.section_names)
+        abort = False
+        logger.debug('Verifying and downloading plugins user by configuration sections: %s' % ', '.join(section_names))
+        for sec in sorted(section_names):
+            section = self.objztpJson.ztpDict.get(sec)
+            t = getTimestamp()
+            try:
+                # Retrieve individual section's progress
+                sec_status = section.get('status')
+                download_check = section.get('pre-ztp-plugin-download')
+                if sec_status == 'BOOT' and download_check:
+                    logger.info('Verifying and downloading plugin used by the configuration section %s.' % (sec))
+                    updateActivity('Verifying and downloading plugin used by the configuration section %s' % sec)
+                    # Get the appropriate plugin to be used for this configuration section
+                    plugin = self.objztpJson.plugin(sec)
+                    if plugin is None:
+                        # Mark section status as failed
+                        section['error'] = 'Unable to find or download requested plugin'
+                        section['start-timestamp'] = t
+                        self.objztpJson.updateStatus(section, 'FAILED')
+                        if not section.get('ignore-result'):
+                            abort = True
+
+            except Exception as e:
+                    logger.error('Exception [%s] encountered while downloading plugin for configuration section %s. Marking it as FAILED.' % (str(e), sec))
+                    section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e))
+                    section['start-timestamp'] = t
+                    self.objztpJson.updateStatus(section, 'FAILED')
+                    if not section.get('ignore-result'):
+                        abort = True
+        return abort
+
     def __processConfigSections(self):
         '''!
          Process and execute individual configuration sections defined in ZTP JSON. Plugin for each
          configuration section is resolved and executed. Configuration section data is provided as
          command line argument to the plugin. Each and every section is processed before this function
          returns.
-
+           @return  False - If error encountered processing configuration sections and request restarting ZTP
+                    True  - If processing of configuration sections has been completed
         '''
 
         # Obtain a copy of the list of configuration sections
@@ -447,6 +488,9 @@ def __processConfigSections(self):
         # set temporary flags
         abort = False
         sort = True
+        if self.__downloadPlugins():
+            logger.info('Halting ZTP as download of one or more plugins FAILED.')
+            return False
 
         logger.debug('Processing configuration sections: %s' % ', '.join(section_names))
         # Loop through each sections till all of them are processed
@@ -537,6 +581,7 @@ def __processConfigSections(self):
 
                 # Check reboot on result flags
                 self.__rebootAction(section)
+        return True
 
     def __processZTPJson(self):
         '''!
@@ -598,32 +643,40 @@ def __processZTPJson(self):
         self.__loadZTPProfile("resume")
 
         # Process available configuration sections in ZTP JSON
-        self.__processConfigSections()
-
-        # Determine ZTP result
-        self.__evalZTPResult()
-
-        # Check restart ZTP condition
-        # ZTP result is failed and restart-ztp-on-failure is set  or
-        _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \
-                        self.objztpJson['restart-ztp-on-failure'] == True)
-
-        # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set
-        _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \
-                           self.objztpJson['restart-ztp-no-config'] == True and \
-                           self.objztpJson['config-fallback'] == False and
-                           os.path.isfile(getCfg('config-db-json')) is False )
+        _processing_completed = self.__processConfigSections()
 
+        # In test mode always mark processing as completed
+        if self.test_mode:
+            _processing_completed = True
+
+        _restart_ztp_missing_config = False
+        _restart_ztp_on_failure = False
+        if _processing_completed:
+            # Determine ZTP result
+            self.__evalZTPResult()
+
+            # Check restart ZTP condition
+            # ZTP result is failed and restart-ztp-on-failure is set  or
+            _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \
+                            self.objztpJson['restart-ztp-on-failure'] == True)
+
+            # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set
+            _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \
+                               self.objztpJson['restart-ztp-no-config'] == True and \
+                               self.objztpJson['config-fallback'] == False and
+                               os.path.isfile(getCfg('config-db-json')) is False )
         # Mark ZTP for restart
-        if _restart_ztp_missing_config or _restart_ztp_on_failure:
+        if not _processing_completed or _restart_ztp_missing_config or _restart_ztp_on_failure:
             os.remove(getCfg('ztp-json'))
             if os.path.isfile(getCfg('ztp-json-shadow')):
-               os.remove(getCfg('ztp-json-shadow'))
+                os.remove(getCfg('ztp-json-shadow'))
             self.objztpJson = None
             # Remove startup-config file to obtain a new one through ZTP
             if getCfg('monitor-startup-config') is True and os.path.isfile(getCfg('config-db-json')):
                 os.remove(getCfg('config-db-json'))
-            if _restart_ztp_missing_config:
+            if not _processing_completed:
+                return ("restart", "Restarting ZTP due to error processing configuration sections")
+            elif _restart_ztp_missing_config:
                 return ("restart", "ZTP completed but startup configuration '%s' not found" % (getCfg('config-db-json')))
             elif _restart_ztp_on_failure:
                 return ("restart", "ZTP completed with FAILED status")

tests/test_ZTPJson.py

@@ -191,6 +191,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir):
                     "source": "/tmp/test_firmware_%s.json"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
@@ -199,6 +200,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir):
         "config-fallback": false,
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "restart-ztp-no-config": true,
@@ -305,6 +307,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir):
                     "source": "http://localhost:2000/ztp/scripts/post_install.sh"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
@@ -313,6 +316,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir):
         "config-fallback": false,
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "restart-ztp-no-config": true,
@@ -459,6 +463,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir):
                     "source": "file:///tmp/test_firmware.sh"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
@@ -467,6 +472,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir):
         "config-fallback": false,
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "restart-ztp-no-config": true,
@@ -526,13 +532,15 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir):
                     "destination": "/tmp/firmware_check.sh"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
             "timestamp": "2019-04-18 19:49:49"
         },
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "status": "BOOT",
@@ -682,13 +690,15 @@ def test_ztp_url_could_not_interpreted(self, tmpdir):
                     "source": "http://localhost:2000/ztp/scripts/post_install.sh"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
             "timestamp": "2019-04-18 19:49:49"
         },
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "status": None,
@@ -765,13 +775,15 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir):
                     "source": "http://localhost:2000/ztp/scripts/post_install.sh"
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
             "timestamp": "2019-04-18 19:49:49"
         },
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "status": None,
@@ -813,13 +825,15 @@ def test_ztp_dynamic_url_download(self, tmpdir):
                       }
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
             "timestamp": "2019-04-18 19:49:49"
         },
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "status": "BOOT",
@@ -874,13 +888,15 @@ def test_ztp_dynamic_url_download_2(self, tmpdir):
                       }
                 }
             },
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
             "timestamp": "2019-04-18 19:49:49"
         },
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "status": "BOOT",
@@ -927,6 +943,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir):
             "halt-on-failure": false,
             "ignore-result": false,
             "plugin": "graphservice",
+            "pre-ztp-plugin-download": true,
             "reboot-on-failure": false,
             "reboot-on-success": false,
             "status": "BOOT",
@@ -935,6 +952,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir):
         "config-fallback": false,
         "halt-on-failure": false,
         "ignore-result": false,
+        "pre-ztp-plugin-download": true,
         "reboot-on-failure": false,
         "reboot-on-success": false,
         "restart-ztp-no-config": true,

tests/test_ztp_engine.py

@@ -448,6 +448,93 @@ def test_ztp_json_ignore_fail(self):
         self.cfgSet('monitor-startup-config', True)
         self.cfgSet('restart-ztp-no-config', True)
 
+    def test_ztp_json_invalid_plugins(self):
+        '''!
+          Simple ZTP test with 3-sections, invalid plugin in the second section, ZTP Failed
+        '''
+        content = """{
+    "ztp": {
+        "0001-test-plugin": {
+           "message" : "0001-test-plugin",
+           "message-file" : "/etc/ztp.results"
+        },
+        "0002-invalid-plugin": {
+            "pre-ztp-plugin-download" : false
+        },
+        "0003-test-plugin": {
+            "pre-ztp-plugin-download" : false,
+            "message" : "0003-test-plugin",
+            "message-file" : "/etc/ztp.results"
+        }
+    }
+}"""
+        expected_result = """0001-test-plugin
+0003-test-plugin
+"""
+        self.__init_ztp_data()
+        self.cfgSet('monitor-startup-config', False)
+        self.cfgSet('restart-ztp-no-config', False)
+        self.__write_file("/tmp/ztp_input.json", content)
+        self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json")
+        runCommand(COVERAGE + ZTP_ENGINE_CMD)
+        runCommand(COVERAGE + ZTP_CMD + ' status -v')
+        os.remove("/tmp/ztp_input.json")
+        objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4)
+        assert(jsonDict.get('ztp').get('status') == 'FAILED')
+        assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS')
+        assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED')
+        assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin')
+        result = self.__read_file("/etc/ztp.results")
+        assert(result == expected_result)
+        self.cfgSet('monitor-startup-config', True)
+        self.cfgSet('restart-ztp-no-config', True)
+
+    def test_ztp_json_invalid_plugins_ignore_result(self):
+        '''!
+          Simple ZTP test with 3-sections, invalid plugin in the second section but result ignored,
+          ZTP Success
+        '''
+        content = """{
+    "ztp": {
+        "0001-test-plugin": {
+           "message" : "0001-test-plugin",
+           "message-file" : "/etc/ztp.results"
+        },
+        "0002-invalid-plugin": {
+           "plugin" : {
+               "url" : "file:///no-such-plugin"
+           },
+           "pre-ztp-plugin-download" : false,
+           "ignore-result" : true
+        },
+        "0003-test-plugin": {
+            "pre-ztp-plugin-download" : false,
+            "message" : "0003-test-plugin",
+            "message-file" : "/etc/ztp.results"
+        }
+    }
+}"""
+        expected_result = """0001-test-plugin
+0003-test-plugin
+"""
+        self.__init_ztp_data()
+        self.cfgSet('monitor-startup-config', False)
+        self.cfgSet('restart-ztp-no-config', False)
+        self.__write_file("/tmp/ztp_input.json", content)
+        self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json")
+        runCommand(COVERAGE + ZTP_ENGINE_CMD)
+        runCommand(COVERAGE + ZTP_CMD + ' status -v')
+        os.remove("/tmp/ztp_input.json")
+        objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4)
+        assert(jsonDict.get('ztp').get('status') == 'SUCCESS')
+        assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS')
+        assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED')
+        assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin')
+        result = self.__read_file("/etc/ztp.results")
+        assert(result == expected_result)
+        self.cfgSet('monitor-startup-config', True)
+        self.cfgSet('restart-ztp-no-config', True)
+
     def test_ztp_json_ignore_ztp_success(self):
         '''!
           Simple ZTP test with 3-sections, Failure in 3 sections but ignore-result set in 2sections, ignore-result set in ztp, ZTP Success