From fd77e90f22d93d4832d2971af75dc60f13b50033 Mon Sep 17 00:00:00 2001 From: blagoja Date: Fri, 31 Jan 2025 16:07:17 +0100 Subject: [PATCH 1/9] Implement IResourceController --- ckanext/afucn/plugin.py | 8 ++++++++ ckanext/afucn/subresource.py | 7 +++++++ 2 files changed, 15 insertions(+) create mode 100644 ckanext/afucn/subresource.py diff --git a/ckanext/afucn/plugin.py b/ckanext/afucn/plugin.py index 75f0169..af2a0a5 100644 --- a/ckanext/afucn/plugin.py +++ b/ckanext/afucn/plugin.py @@ -1,12 +1,14 @@ import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit from ckan.lib.plugins import DefaultTranslation +from ckanext.afucn.subresource import create_subresource class AfucnPlugin(plugins.SingletonPlugin, DefaultTranslation): plugins.implements(plugins.IConfigurer) plugins.implements(plugins.IFacets) plugins.implements(plugins.ITranslation) + plugins.implements(plugins.IResourceController, inherit=True) # IConfigurer @@ -55,3 +57,9 @@ def organization_facets(self, facets_dict, organization_type, package_type): facets_dict['license_id'] = plugins.toolkit._('License') return facets_dict + + # IResourceController + + def after_resource_create(self, context, resource_dict): + create_subresource(resource_dict) + return diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py new file mode 100644 index 0000000..8a48917 --- /dev/null +++ b/ckanext/afucn/subresource.py @@ -0,0 +1,7 @@ +def create_subresource(resource_dict): + + print("======================================================================") + print(resource_dict) + print("======================================================================") + + return \ No newline at end of file From 7c3051802ed2baca110c0134f8343ffc0c07020a Mon Sep 17 00:00:00 2001 From: blagoja Date: Mon, 3 Feb 2025 10:36:28 +0100 Subject: [PATCH 2/9] Test prints --- ckanext/afucn/plugin.py | 2 +- ckanext/afucn/subresource.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ckanext/afucn/plugin.py b/ckanext/afucn/plugin.py index af2a0a5..41cb9c7 100644 --- a/ckanext/afucn/plugin.py +++ b/ckanext/afucn/plugin.py @@ -61,5 +61,5 @@ def organization_facets(self, facets_dict, organization_type, package_type): # IResourceController def after_resource_create(self, context, resource_dict): - create_subresource(resource_dict) + create_subresource(context, resource_dict) return diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index 8a48917..a5d03a4 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -1,7 +1,16 @@ -def create_subresource(resource_dict): +import ckan.plugins.toolkit as tk +from ckan.common import g + +def create_subresource(context, resource_dict): + + x = tk.get_action("status_show")(context) + print("======================================================================") + print(g.user) print("======================================================================") print(resource_dict) print("======================================================================") + print(x) + print("======================================================================") - return \ No newline at end of file + return From 9a54cfd79eeb1def243beed1f7c76ace127be9a1 Mon Sep 17 00:00:00 2001 From: blagoja Date: Mon, 3 Feb 2025 16:37:25 +0100 Subject: [PATCH 3/9] Test add resource action --- ckanext/afucn/subresource.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index a5d03a4..d67a850 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -4,13 +4,18 @@ def create_subresource(context, resource_dict): - x = tk.get_action("status_show")(context) + local_file_path = '/home/blagoja/Downloads/sample_data/sample.txt' + + with open(local_file_path, 'rb') as file_content: + data_dict = { + 'package_id': resource_dict['package_id'], + 'name': 'Subresource', + 'upload': file_content, + } + + # x = tk.get_action("resource_create")(context, data_dict) + print("======================================================================") print(g.user) print("======================================================================") - print(resource_dict) - print("======================================================================") - print(x) - print("======================================================================") - - return + return From e33ecd7a8fc43c3c1eb7f594da69a641d72c8e33 Mon Sep 17 00:00:00 2001 From: blagoja Date: Tue, 4 Feb 2025 22:58:57 +0100 Subject: [PATCH 4/9] Add subresource based on resource format --- ckanext/afucn/subresource.py | 39 +++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index d67a850..00c8895 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -1,21 +1,42 @@ import ckan.plugins.toolkit as tk from ckan.common import g +from ckan.common import config +import os +import logging +from pathlib import Path +from shutil import copyfile + +log = logging.getLogger(__name__) + +try: + storage_path = config.get('ckan.storage_path') +except: + log.critical('''Please specify a ckan.storage_path in your config + for your uploads''') def create_subresource(context, resource_dict): - local_file_path = '/home/blagoja/Downloads/sample_data/sample.txt' + if resource_dict['format'] == 'CSV': - with open(local_file_path, 'rb') as file_content: data_dict = { 'package_id': resource_dict['package_id'], - 'name': 'Subresource', - 'upload': file_content, + 'name': 'test.jpg', + 'url': 'test.jpg', + 'url_type': 'upload', } - # x = tk.get_action("resource_create")(context, data_dict) + x = tk.get_action("resource_create")(context, data_dict) + upload_path = storage_path + '/resources/' + x['id'][0:3] + "/" + x['id'][3:6] + upload_filename = x['id'][6:] + filepath = Path(os.path.join(upload_path, upload_filename)) + filepath.parent.mkdir(parents=True, exist_ok=True) + copyfile('/home/blagoja/Downloads/demo/test.jpeg', filepath) + return + + else: + print("======================================================================") + print(g.user) + print("======================================================================") + return - print("======================================================================") - print(g.user) - print("======================================================================") - return From 15d134ea5cf7aec905ee0335024022f63188589e Mon Sep 17 00:00:00 2001 From: blagoja Date: Wed, 5 Feb 2025 21:26:55 +0100 Subject: [PATCH 5/9] Check subresource create by resource name --- ckanext/afucn/subresource.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index 00c8895..af39ca0 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -1,5 +1,4 @@ import ckan.plugins.toolkit as tk -from ckan.common import g from ckan.common import config import os import logging @@ -17,7 +16,18 @@ def create_subresource(context, resource_dict): - if resource_dict['format'] == 'CSV': + package_data = { + 'id': resource_dict['package_id'], + } + + resources_list = tk.get_action("package_show")(context, package_data)['resources'] + + names_list = [] + + for i in resources_list: + names_list.append(i['name']) + + if 'test.jpg' not in names_list: data_dict = { 'package_id': resource_dict['package_id'], @@ -36,7 +46,7 @@ def create_subresource(context, resource_dict): else: print("======================================================================") - print(g.user) + print('subresource not created') print("======================================================================") return From bab8a937a0919299b45107214fd888009aefacc3 Mon Sep 17 00:00:00 2001 From: blagoja Date: Fri, 7 Feb 2025 10:14:33 +0100 Subject: [PATCH 6/9] Add geojson subresource --- ckanext/afucn/subresource.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index af39ca0..09c0df7 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -4,6 +4,9 @@ import logging from pathlib import Path from shutil import copyfile +import openpyxl +import shutil +import tempfile log = logging.getLogger(__name__) @@ -14,6 +17,24 @@ for your uploads''') +def extract_map(filename): + # filename = '/home/blagoja/Downloads/test_map.xlsx' + wb = openpyxl.load_workbook(filename) + ws = wb['Sheet1'] + + for idx, row in enumerate(ws.iter_rows()): + geojson_text = row[1].value + + f = tempfile.NamedTemporaryFile(mode='w+t', delete=False) + f.write(geojson_text) + file_name = f.name + f.close() + shutil.copy(file_name, 'bar.geojson') + os.remove(file_name) + + return + + def create_subresource(context, resource_dict): package_data = { @@ -22,26 +43,27 @@ def create_subresource(context, resource_dict): resources_list = tk.get_action("package_show")(context, package_data)['resources'] + subresource_name = resource_dict['name'].rsplit('.', 1)[0] + '.geojson' names_list = [] - + for i in resources_list: names_list.append(i['name']) - if 'test.jpg' not in names_list: + if subresource_name not in names_list: data_dict = { 'package_id': resource_dict['package_id'], - 'name': 'test.jpg', - 'url': 'test.jpg', + 'name': subresource_name, + 'url': subresource_name, 'url_type': 'upload', } - x = tk.get_action("resource_create")(context, data_dict) upload_path = storage_path + '/resources/' + x['id'][0:3] + "/" + x['id'][3:6] upload_filename = x['id'][6:] filepath = Path(os.path.join(upload_path, upload_filename)) filepath.parent.mkdir(parents=True, exist_ok=True) - copyfile('/home/blagoja/Downloads/demo/test.jpeg', filepath) + copyfile('/home/blagoja/Downloads/demo/test.geojson', filepath) + return else: From 8a98e94c62bf915322d8caa6e80b0ea05300c2b1 Mon Sep 17 00:00:00 2001 From: blagoja Date: Sun, 9 Feb 2025 00:24:49 +0100 Subject: [PATCH 7/9] Create geojson from resource --- ckanext/afucn/subresource.py | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index 09c0df7..3310762 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -3,10 +3,12 @@ import os import logging from pathlib import Path -from shutil import copyfile import openpyxl +from openpyxl import load_workbook import shutil import tempfile +from io import BytesIO +import requests log = logging.getLogger(__name__) @@ -17,22 +19,14 @@ for your uploads''') -def extract_map(filename): - # filename = '/home/blagoja/Downloads/test_map.xlsx' - wb = openpyxl.load_workbook(filename) - ws = wb['Sheet1'] - - for idx, row in enumerate(ws.iter_rows()): - geojson_text = row[1].value - - f = tempfile.NamedTemporaryFile(mode='w+t', delete=False) - f.write(geojson_text) - file_name = f.name - f.close() - shutil.copy(file_name, 'bar.geojson') - os.remove(file_name) - - return +def get_excel_from_url(url): + response = requests.get(url) + if response.status_code == 200: + file_content = BytesIO(response.content) + workbook = load_workbook(file_content) + return workbook + else: + return response.text def create_subresource(context, resource_dict): @@ -48,27 +42,41 @@ def create_subresource(context, resource_dict): for i in resources_list: names_list.append(i['name']) - if subresource_name not in names_list: - + data_dict = { 'package_id': resource_dict['package_id'], 'name': subresource_name, 'url': subresource_name, 'url_type': 'upload', } + + filename = '/home/blagoja/Downloads/test_map.xlsx' + wb = openpyxl.load_workbook(filename) + ws = wb['Sheet1'] + + for idx, row in enumerate(ws.iter_rows()): + geojson_text = row[1].value + + f = tempfile.NamedTemporaryFile(mode='w+t', delete=False) + f.write(geojson_text) + file_name = f.name + f.close() + x = tk.get_action("resource_create")(context, data_dict) upload_path = storage_path + '/resources/' + x['id'][0:3] + "/" + x['id'][3:6] upload_filename = x['id'][6:] + Path(upload_path).mkdir(parents=True, exist_ok=True) filepath = Path(os.path.join(upload_path, upload_filename)) - filepath.parent.mkdir(parents=True, exist_ok=True) - copyfile('/home/blagoja/Downloads/demo/test.geojson', filepath) - - return + shutil.copy(file_name, filepath) + os.remove(file_name) + + return resource_dict else: print("======================================================================") print('subresource not created') print("======================================================================") - return + + return resource_dict From 2e9275afa4726256191d00e510c260c41b892367 Mon Sep 17 00:00:00 2001 From: blagoja Date: Sun, 9 Feb 2025 14:48:17 +0100 Subject: [PATCH 8/9] Create map resource from another resource from the dataset --- ckanext/afucn/subresource.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/ckanext/afucn/subresource.py b/ckanext/afucn/subresource.py index 3310762..231d4d5 100644 --- a/ckanext/afucn/subresource.py +++ b/ckanext/afucn/subresource.py @@ -3,12 +3,10 @@ import os import logging from pathlib import Path -import openpyxl from openpyxl import load_workbook import shutil import tempfile -from io import BytesIO -import requests + log = logging.getLogger(__name__) @@ -19,15 +17,11 @@ for your uploads''') -def get_excel_from_url(url): - response = requests.get(url) - if response.status_code == 200: - file_content = BytesIO(response.content) - workbook = load_workbook(file_content) - return workbook - else: - return response.text +def get_resource_path(resource_id): + resource_path = storage_path + "/resources/" + resource_id[0:3] + "/" + resource_id[3:6] + "/" + resource_id[6:] + + return resource_path def create_subresource(context, resource_dict): @@ -51,8 +45,14 @@ def create_subresource(context, resource_dict): 'url_type': 'upload', } - filename = '/home/blagoja/Downloads/test_map.xlsx' - wb = openpyxl.load_workbook(filename) + resource_path = get_resource_path(resource_dict['id']) + resource_copy_path = resource_path + ".xlsx" + + # Copy the file to the same location with .xls extension + shutil.copy(resource_path, resource_copy_path) + + # Load the excel file from copied resource + wb = load_workbook(resource_copy_path) ws = wb['Sheet1'] for idx, row in enumerate(ws.iter_rows()): @@ -63,13 +63,18 @@ def create_subresource(context, resource_dict): file_name = f.name f.close() + # Create the map resource from the excel file and x = tk.get_action("resource_create")(context, data_dict) upload_path = storage_path + '/resources/' + x['id'][0:3] + "/" + x['id'][3:6] upload_filename = x['id'][6:] Path(upload_path).mkdir(parents=True, exist_ok=True) filepath = Path(os.path.join(upload_path, upload_filename)) shutil.copy(file_name, filepath) - os.remove(file_name) + + # remove temp xls resource from resources directory + os.remove(resource_copy_path) + # remove temp subresource file + os.remove(file_name) return resource_dict From ea23ffb38522612574ae5b032d49a017ebfff4fa Mon Sep 17 00:00:00 2001 From: blagoja Date: Mon, 10 Feb 2025 20:42:25 +0100 Subject: [PATCH 9/9] Add config option for subresource --- ckanext/afucn/plugin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ckanext/afucn/plugin.py b/ckanext/afucn/plugin.py index 41cb9c7..2849b01 100644 --- a/ckanext/afucn/plugin.py +++ b/ckanext/afucn/plugin.py @@ -2,6 +2,9 @@ import ckan.plugins.toolkit as toolkit from ckan.lib.plugins import DefaultTranslation from ckanext.afucn.subresource import create_subresource +from ckan.common import config + +subresource = config.get('ckanext.afucn.subresource', False) class AfucnPlugin(plugins.SingletonPlugin, DefaultTranslation): @@ -61,5 +64,6 @@ def organization_facets(self, facets_dict, organization_type, package_type): # IResourceController def after_resource_create(self, context, resource_dict): - create_subresource(context, resource_dict) + if subresource: + create_subresource(context, resource_dict) return