From 88dc031f772916e7165ca75acde1c320afdd933b Mon Sep 17 00:00:00 2001 From: realvudi Date: Mon, 29 Apr 2024 17:28:06 -0700 Subject: [PATCH 01/17] Update to DB qiita.slurm_resource_allocations --- notebooks/resource-allocation/upload_df.py | 72 ++ qiita_db/support_files/populate_test_db.sql | 929 ++++++++++++++++++++ 2 files changed, 1001 insertions(+) create mode 100644 notebooks/resource-allocation/upload_df.py diff --git a/notebooks/resource-allocation/upload_df.py b/notebooks/resource-allocation/upload_df.py new file mode 100644 index 000000000..e74bcce04 --- /dev/null +++ b/notebooks/resource-allocation/upload_df.py @@ -0,0 +1,72 @@ +import pandas as pd + +# Example data loading +filename = './data/jobs_2024-02-21.tsv.gz' +df = pd.read_csv(filename, sep='\t', dtype={'extra_info': str}) + +# Convert string to timedelta, then to total seconds +df['ElapsedRawTime'] = pd.to_timedelta( + df['ElapsedRawTime']).apply( + lambda x: x.total_seconds()) + +cname = "Validate" +sname = "Diversity types - alpha_vector" +df = df[(df.cName == cname) & (df.sName == sname)] + +df['samples'] = df['samples'].fillna(0).astype(int) +df['columns'] = df['columns'].fillna(0).astype(int) +df['input_size'] = df['input_size'].fillna(0).astype(int) +df['MaxRSSRaw'] = df['MaxRSSRaw'].fillna(0).astype(int) +df['ElapsedRawTime'] = df['ElapsedRawTime'].fillna(0).astype(int) + +COL_NAME = 'samples * columns' +df[COL_NAME] = df['samples'] * df['columns'] +columns = ["MaxRSSRaw", "ElapsedRawTime"] +max_rows = [] + +for curr in columns: + # Get the maximum value for 'curr' within each COL_NAME group + max_values = df.groupby(COL_NAME)[curr].transform(max) + # Filter rows where the current column's value + # is the maximum within its group + curr_rows = df[df[curr] == max_values] + max_rows.append(curr_rows) + +filtered_df = pd.concat(max_rows).drop_duplicates().reset_index(drop=True) + +# INSERT INTO qiita.processing_job(processing_job_id, email, command_id, +# command_parameters, processing_job_status_id) +# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', +# 'admin@foo.com', 1, '""'::json, 1); + +# INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, +# columns, input_size, extra_info, memory_used, walltime_used) +# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', +# 327036000, 91); + +# processing_job_id uuid NOT NULL, +# samples integer, +# columns integer, +# input_size bigint, +# extra_info varchar DEFAULT NULL, +# memory_used bigint, +# walltime_used integer, + +res = "" + +for index, row in filtered_df.iterrows(): + res += f"""('{row['QiitaID']}', 'admin@foo.bar', 1, '""'::json, 1),\n""" +res += ";\n" +res += "Split\n" +for index, row in filtered_df.iterrows(): + res += ( + f"('{row['QiitaID']}', {int(row['samples'])}, " + f"{int(row['columns'])}, {int(row['input_size'])}, " + f"'{row['extra_info']}', {int(row['MaxRSSRaw'])}, " + f"{int(row['ElapsedRawTime'])}),\n" + ) + +res += ";\n" + +with open("sql.txt", 'w') as filename: + filename.write(res) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index f8b5466b2..c0a8c7536 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -582,3 +582,932 @@ INSERT INTO qiita.processing_job_workflow_root (processing_job_workflow_id, proc INSERT INTO qiita.parent_processing_job (parent_id, child_id) VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'd19f76ee-274e-4c1b-b3a2-a12d73507c55'); + +-- SPLIT + +INSERT INTO qiita.processing_job(processing_job_id, email, command_id, command_parameters, processing_job_status_id) +VALUES ('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 'admin@foo.bar', 1, '""'::json, 1), +('b0f36550-d97c-4dd5-ba1b-910470062801', 'admin@foo.bar', 1, '""'::json, 1), +('b13eefd1-12b7-4c3e-a380-94ec0a3f9f91', 'admin@foo.bar', 1, '""'::json, 1), +('469c27b7-81c4-4f41-b09d-149260961227', 'admin@foo.bar', 1, '""'::json, 1), +('6f198e33-b4ae-47e9-973b-bd5e0b06c353', 'admin@foo.bar', 1, '""'::json, 1), +('045eb36c-5144-4d54-a621-23eb5dc4d6ef', 'admin@foo.bar', 1, '""'::json, 1), +('041f3278-2752-44f9-8caa-ac6ae80c98e6', 'admin@foo.bar', 1, '""'::json, 1), +('6325e683-b215-4c56-bf5a-cbbc92330067', 'admin@foo.bar', 1, '""'::json, 1), +('03dcca05-0d4f-48c5-bc7a-7ef884be8efc', 'admin@foo.bar', 1, '""'::json, 1), +('8d84685f-97f4-4ea7-8857-2dcec5a20775', 'admin@foo.bar', 1, '""'::json, 1), +('8b714fe7-51d9-492c-9b49-4fe09b595067', 'admin@foo.bar', 1, '""'::json, 1), +('490c0623-4ed7-4064-9ffe-ba5afc0f0aa0', 'admin@foo.bar', 1, '""'::json, 1), +('c8301f3d-a4fe-407a-9aec-b9dfee0cc2fd', 'admin@foo.bar', 1, '""'::json, 1), +('561b9676-604a-4cdb-833f-a4a9b3afef42', 'admin@foo.bar', 1, '""'::json, 1), +('8bc35697-f3df-42f6-8a34-830d42000bd6', 'admin@foo.bar', 1, '""'::json, 1), +('474a4a8d-0963-4fe6-9da4-74e520532be1', 'admin@foo.bar', 1, '""'::json, 1), +('d4d03b6f-12f1-4f47-9ad4-b5597a3e5a16', 'admin@foo.bar', 1, '""'::json, 1), +('27d95aba-9b36-40c8-8d17-7ade170172aa', 'admin@foo.bar', 1, '""'::json, 1), +('a8f00090-6981-4fc3-bc37-208f01e6dceb', 'admin@foo.bar', 1, '""'::json, 1), +('d7562d61-7886-45b5-9fed-90fef6636e6f', 'admin@foo.bar', 1, '""'::json, 1), +('e4071da6-550d-469b-91dc-f6cda658df80', 'admin@foo.bar', 1, '""'::json, 1), +('c126e507-5f13-485f-9166-775ee7228f12', 'admin@foo.bar', 1, '""'::json, 1), +('d73906a4-f2c7-4d03-bd55-d637365ec3bc', 'admin@foo.bar', 1, '""'::json, 1), +('167efba9-a88e-493c-a2c6-0647556ba6dd', 'admin@foo.bar', 1, '""'::json, 1), +('2662c176-d042-4254-bc7e-20c092fc3af1', 'admin@foo.bar', 1, '""'::json, 1), +('54b19d0d-fa47-4c37-a4ea-e2be33d54c5b', 'admin@foo.bar', 1, '""'::json, 1), +('8b3e969e-78c4-4a09-9767-4aa72d99ee8c', 'admin@foo.bar', 1, '""'::json, 1), +('bef26557-60db-4987-b0bf-e9e0be77da37', 'admin@foo.bar', 1, '""'::json, 1), +('adcf6cb4-780b-4a9d-bb7c-cd2bf12ac201', 'admin@foo.bar', 1, '""'::json, 1), +('54464698-5860-4543-9376-d3d89722848f', 'admin@foo.bar', 1, '""'::json, 1), +('9a8b3ea7-36d7-46f7-8602-1b39c89f5ee4', 'admin@foo.bar', 1, '""'::json, 1), +('b57c1bc6-a805-463d-87fc-b76a186e5a63', 'admin@foo.bar', 1, '""'::json, 1), +('0c0579c8-dc34-4211-ad87-c453e9cb10bf', 'admin@foo.bar', 1, '""'::json, 1), +('04357795-299a-4900-9fdf-debf3be2ccc7', 'admin@foo.bar', 1, '""'::json, 1), +('fa1617ad-daf8-496d-937f-5ad9526f57d2', 'admin@foo.bar', 1, '""'::json, 1), +('056dd777-1c93-424a-ad7a-fbac4ff01e6b', 'admin@foo.bar', 1, '""'::json, 1), +('1d056722-c08c-45e7-abbf-794abf85b2ba', 'admin@foo.bar', 1, '""'::json, 1), +('a4e4457e-e191-4951-81d2-a1bd8afa7bb9', 'admin@foo.bar', 1, '""'::json, 1), +('2bd5eedb-f4bd-4221-a31d-7a82ed1ed85c', 'admin@foo.bar', 1, '""'::json, 1), +('59e6279e-5a92-4ad5-81cd-d83892eb35e3', 'admin@foo.bar', 1, '""'::json, 1), +('594e5a37-e00e-44e3-8baf-a5d0fbd8d1c5', 'admin@foo.bar', 1, '""'::json, 1), +('40e16ce2-d280-423b-908e-f74877751bb8', 'admin@foo.bar', 1, '""'::json, 1), +('df964c5d-d204-475d-bf37-c4281c830e36', 'admin@foo.bar', 1, '""'::json, 1), +('f1ff0a3f-5004-49d4-a781-fedab076f682', 'admin@foo.bar', 1, '""'::json, 1), +('d5b6766a-5697-4283-b7f5-ded6295e539d', 'admin@foo.bar', 1, '""'::json, 1), +('6e98873c-0c99-47f0-ae6f-fc12092c1c5d', 'admin@foo.bar', 1, '""'::json, 1), +('b816de4b-42f6-4495-8a05-7cb7faefcf3e', 'admin@foo.bar', 1, '""'::json, 1), +('2e196649-1641-4ced-bf74-f24db6123d66', 'admin@foo.bar', 1, '""'::json, 1), +('ca8271ce-ff25-4a22-8309-149396d20d38', 'admin@foo.bar', 1, '""'::json, 1), +('38496999-f362-45d1-a9e6-6c2ed7ad1847', 'admin@foo.bar', 1, '""'::json, 1), +('dedeab8b-ed71-4d3d-87c1-37f607e19e5d', 'admin@foo.bar', 1, '""'::json, 1), +('997ad2aa-59ff-4346-805c-cbe1a41bac00', 'admin@foo.bar', 1, '""'::json, 1), +('ff866805-7add-432f-90a8-15ab4b793cdd', 'admin@foo.bar', 1, '""'::json, 1), +('ef8a5faa-fc7b-4a09-a45a-7aa4d3dbb1f8', 'admin@foo.bar', 1, '""'::json, 1), +('b0db6327-e4ff-4dd5-85ae-b8a060f8ed9f', 'admin@foo.bar', 1, '""'::json, 1), +('900537c6-a4bb-4f20-9674-e0fd4851c10a', 'admin@foo.bar', 1, '""'::json, 1), +('3fe82723-af0e-4df0-a3bf-32223b202dd5', 'admin@foo.bar', 1, '""'::json, 1), +('6b22c272-f72c-4102-9abb-afff9d1c7d42', 'admin@foo.bar', 1, '""'::json, 1), +('6738f416-1ce1-4cac-90ef-ead48b9b0ace', 'admin@foo.bar', 1, '""'::json, 1), +('6ed39d69-1c01-42c1-8249-b8abf83c3608', 'admin@foo.bar', 1, '""'::json, 1), +('d00fd8ee-7d65-469c-af2a-ca00681d224b', 'admin@foo.bar', 1, '""'::json, 1), +('42244303-b4a1-43d3-9041-f4f811691b41', 'admin@foo.bar', 1, '""'::json, 1), +('4ac590b9-5733-4e20-b657-3b13d9b30a4f', 'admin@foo.bar', 1, '""'::json, 1), +('1409a826-8176-49db-84f9-04d4cac0db89', 'admin@foo.bar', 1, '""'::json, 1), +('21917565-5c24-4b9e-947e-0a7fc18b89dc', 'admin@foo.bar', 1, '""'::json, 1), +('cc4db5ef-c3d3-4067-98be-ee90a584a95a', 'admin@foo.bar', 1, '""'::json, 1), +('12f627b7-19aa-4583-bce6-a7f7d4dd66a2', 'admin@foo.bar', 1, '""'::json, 1), +('0c6dc99d-d20b-4381-a405-8813129b81ee', 'admin@foo.bar', 1, '""'::json, 1), +('449668bf-3294-42eb-9cef-d0c007b9b27f', 'admin@foo.bar', 1, '""'::json, 1), +('ee4c2079-e187-4a2f-ae9a-a95795a9d5fe', 'admin@foo.bar', 1, '""'::json, 1), +('0f0d6f75-6ba3-4265-b53e-cc53a18e52b8', 'admin@foo.bar', 1, '""'::json, 1), +('d63bdad8-ace7-4d92-a8a7-add8620460d3', 'admin@foo.bar', 1, '""'::json, 1), +('5a856042-6a60-4f9d-b256-d03295e2d7f9', 'admin@foo.bar', 1, '""'::json, 1), +('dd9969e8-9efe-4781-af4a-6f2f8ff70808', 'admin@foo.bar', 1, '""'::json, 1), +('37cff1b2-c11e-4511-8625-84def4635d84', 'admin@foo.bar', 1, '""'::json, 1), +('36f312b9-5055-443a-b07a-2ab09e8f8cfa', 'admin@foo.bar', 1, '""'::json, 1), +('de207a0d-932f-4a81-9412-5ff8baa9cdd5', 'admin@foo.bar', 1, '""'::json, 1), +('b567274d-4631-4aee-85df-7d6603c20cfe', 'admin@foo.bar', 1, '""'::json, 1), +('94d07841-1b75-42a1-9407-af6a341769b1', 'admin@foo.bar', 1, '""'::json, 1), +('d83a1dee-1451-4ea3-bd42-3881f1fa4e3e', 'admin@foo.bar', 1, '""'::json, 1), +('3c736f54-7d2d-4cd6-9c45-8cce37ce5482', 'admin@foo.bar', 1, '""'::json, 1), +('e600feed-52cc-4d8f-b28d-c6283dcde944', 'admin@foo.bar', 1, '""'::json, 1), +('41624971-b40b-4e03-bb12-cce149cd6eac', 'admin@foo.bar', 1, '""'::json, 1), +('9f769164-d25c-4e7d-b637-79f3489798c5', 'admin@foo.bar', 1, '""'::json, 1), +('0f382217-32ed-4ea6-b7ab-c98c66e8305f', 'admin@foo.bar', 1, '""'::json, 1), +('24d2cf39-25a1-486e-a763-f2639674f4b4', 'admin@foo.bar', 1, '""'::json, 1), +('b303df36-67d8-4883-a109-294b61ccbe23', 'admin@foo.bar', 1, '""'::json, 1), +('b46d8a4d-b969-4226-8719-6708c3755775', 'admin@foo.bar', 1, '""'::json, 1), +('07b691e7-a940-4257-bea8-ebba6bc47637', 'admin@foo.bar', 1, '""'::json, 1), +('9d624992-01fe-4ece-a7ba-ebf54da0eb7b', 'admin@foo.bar', 1, '""'::json, 1), +('0aba4000-8b6d-47c1-a4fc-7b6a2b419b67', 'admin@foo.bar', 1, '""'::json, 1), +('4635cab1-b1a7-4d6a-bb75-d95f0e1fd682', 'admin@foo.bar', 1, '""'::json, 1), +('b2602d9e-7ce6-43e2-a4b3-3962f812a80a', 'admin@foo.bar', 1, '""'::json, 1), +('f8d9c8b3-1e4f-4371-9d86-74778f7145a7', 'admin@foo.bar', 1, '""'::json, 1), +('490bf28c-7f5b-428f-9127-c1ac19d611fc', 'admin@foo.bar', 1, '""'::json, 1), +('a69254f1-d5d4-4f3f-991f-c197364b8187', 'admin@foo.bar', 1, '""'::json, 1), +('702f4828-f604-44b8-a7b0-9684e77a0279', 'admin@foo.bar', 1, '""'::json, 1), +('0d793328-c4c5-4739-b2f1-40afb4de55c8', 'admin@foo.bar', 1, '""'::json, 1), +('7227a61e-59af-4b96-8919-efe13839889c', 'admin@foo.bar', 1, '""'::json, 1), +('57e351db-c504-42dd-a699-fe405c1b5a27', 'admin@foo.bar', 1, '""'::json, 1), +('88bbeb39-2fc5-4a46-8bd9-ca28a2fedef3', 'admin@foo.bar', 1, '""'::json, 1), +('63a0101c-13b0-4c73-b4d7-00a33ae0f9ab', 'admin@foo.bar', 1, '""'::json, 1), +('e3c17c93-5993-49fc-891a-9c9d5b9aa43c', 'admin@foo.bar', 1, '""'::json, 1), +('6fd1fc5f-fce2-43c6-909d-8041a075ac3a', 'admin@foo.bar', 1, '""'::json, 1), +('8ea8b437-819b-4624-b15c-a653b53c0a2e', 'admin@foo.bar', 1, '""'::json, 1), +('c4ee461b-8d52-4338-8fa2-8a22b538456b', 'admin@foo.bar', 1, '""'::json, 1), +('1eaa62de-c3ae-414d-9168-7fb5ece89949', 'admin@foo.bar', 1, '""'::json, 1), +('6a63a491-c4c1-4176-9827-9460526ed319', 'admin@foo.bar', 1, '""'::json, 1), +('733a21cf-e969-4a17-b888-820f2f9475a1', 'admin@foo.bar', 1, '""'::json, 1), +('9a1e07e0-bac0-46e1-8b72-86a730f47c12', 'admin@foo.bar', 1, '""'::json, 1), +('6ca8ca71-dbb7-4347-adb3-9ab02b69ef9f', 'admin@foo.bar', 1, '""'::json, 1), +('07272b4e-88bf-4adc-9953-4b1f4cb3764f', 'admin@foo.bar', 1, '""'::json, 1), +('8b9f9e1e-2e52-432b-8528-53f8b25ea81e', 'admin@foo.bar', 1, '""'::json, 1), +('6395bd2a-fd1b-448b-8771-f07a77958d84', 'admin@foo.bar', 1, '""'::json, 1), +('b05bb0d8-a8bd-4634-82f9-47c52ccd83e3', 'admin@foo.bar', 1, '""'::json, 1), +('38ee009a-7c6a-42a2-a05f-cd3f5286cd9d', 'admin@foo.bar', 1, '""'::json, 1), +('d0799585-1e37-4134-b71f-481dc0d2be72', 'admin@foo.bar', 1, '""'::json, 1), +('2d5a21fd-1cdb-4ee6-8e08-584bf99ef40a', 'admin@foo.bar', 1, '""'::json, 1), +('a4aee4f4-91d5-46e1-9280-87b8ba7eeef9', 'admin@foo.bar', 1, '""'::json, 1), +('7e361763-962b-4f8d-8c63-1a329428f89f', 'admin@foo.bar', 1, '""'::json, 1), +('0abb0be4-b58e-4318-8624-4fc055f9d82d', 'admin@foo.bar', 1, '""'::json, 1), +('3497cd39-99dc-429e-8e10-67ecd7b2774a', 'admin@foo.bar', 1, '""'::json, 1), +('0ca036c1-ceb5-4c33-993b-d8511a96208b', 'admin@foo.bar', 1, '""'::json, 1), +('a22718ce-6e93-420b-89d6-cf2ddf104164', 'admin@foo.bar', 1, '""'::json, 1), +('278179bb-f481-4d50-aced-5485993699a1', 'admin@foo.bar', 1, '""'::json, 1), +('80272c6a-4312-4c93-9905-390e62a3d201', 'admin@foo.bar', 1, '""'::json, 1), +('d5ab9552-64e1-43b2-bf46-5065612c2cda', 'admin@foo.bar', 1, '""'::json, 1), +('af467934-f4db-464e-8ef1-b6bbbbe88667', 'admin@foo.bar', 1, '""'::json, 1), +('cfa62dd9-c6e0-48a2-9633-ee4d813d4f51', 'admin@foo.bar', 1, '""'::json, 1), +('bd37012e-0b22-4e3a-b0ae-eba006d87749', 'admin@foo.bar', 1, '""'::json, 1), +('d11bb684-448c-4916-a0f8-cd5748061638', 'admin@foo.bar', 1, '""'::json, 1), +('46cb1988-f5b7-4a27-92be-dd8b7c7289db', 'admin@foo.bar', 1, '""'::json, 1), +('dd000d70-ee4b-46a6-9e1f-26fd85b6df3c', 'admin@foo.bar', 1, '""'::json, 1), +('a12d7860-89d0-4640-a4a6-73738b47408a', 'admin@foo.bar', 1, '""'::json, 1), +('aae1e730-254d-401b-884d-d9f6965203f6', 'admin@foo.bar', 1, '""'::json, 1), +('66e2dbf0-d223-48b6-bf1e-45a40f87d1b1', 'admin@foo.bar', 1, '""'::json, 1), +('be2e6e7d-7992-4407-a7c1-aced14e6fa12', 'admin@foo.bar', 1, '""'::json, 1), +('671071f2-380c-42a3-88f6-a1642c07dc36', 'admin@foo.bar', 1, '""'::json, 1), +('5db2e298-251c-409f-81a2-844b9a2b7697', 'admin@foo.bar', 1, '""'::json, 1), +('9b0e1a4a-97a6-47fe-aa2a-6800d72fa611', 'admin@foo.bar', 1, '""'::json, 1), +('9553c043-e844-4c9f-8367-eb5d7d51772c', 'admin@foo.bar', 1, '""'::json, 1), +('78c3db71-ed10-457d-b048-d8142c535e14', 'admin@foo.bar', 1, '""'::json, 1), +('60a72e37-28f8-4288-81eb-830cb2924a03', 'admin@foo.bar', 1, '""'::json, 1), +('520e37c3-6e64-4fce-8a88-cceac2b3b5b8', 'admin@foo.bar', 1, '""'::json, 1), +('596f24ae-49e1-4a22-bbee-0b53b1bc4b26', 'admin@foo.bar', 1, '""'::json, 1), +('7fce7f74-0c73-4fc2-b5bc-45f570014054', 'admin@foo.bar', 1, '""'::json, 1), +('0061bb8f-219c-4e53-8ea5-137799e5fc3c', 'admin@foo.bar', 1, '""'::json, 1), +('09c27409-4fb5-415c-93e3-7e2233e0109e', 'admin@foo.bar', 1, '""'::json, 1), +('1a95492b-e852-413b-8450-72eb0adcd577', 'admin@foo.bar', 1, '""'::json, 1), +('c6315300-f68e-4bf7-b7d2-778c3409f346', 'admin@foo.bar', 1, '""'::json, 1), +('517dd44b-61bc-47ba-a7c8-c189378f6d73', 'admin@foo.bar', 1, '""'::json, 1), +('c167f049-4d43-4409-84be-233b44981bb8', 'admin@foo.bar', 1, '""'::json, 1), +('2c4290f6-e59f-438f-9664-a22810b495a0', 'admin@foo.bar', 1, '""'::json, 1), +('e7a1816e-c98c-4ae9-854c-b034d4dc77cf', 'admin@foo.bar', 1, '""'::json, 1), +('e5a06530-b313-4b75-ad14-5bceeb690d5a', 'admin@foo.bar', 1, '""'::json, 1), +('8d671718-8686-4cc5-899f-178a2fba763e', 'admin@foo.bar', 1, '""'::json, 1), +('ef88e6d6-affc-4511-8842-af8f32c1b49e', 'admin@foo.bar', 1, '""'::json, 1), +('a1e85eec-0ed6-43ad-b0b0-d50fa4a26ed8', 'admin@foo.bar', 1, '""'::json, 1), +('79baff1d-1f54-429f-91b2-4bc4e6ed5c18', 'admin@foo.bar', 1, '""'::json, 1), +('a318ddfc-153d-4462-b7f9-963f7b3a11b2', 'admin@foo.bar', 1, '""'::json, 1), +('8d376621-fad2-4329-9a54-5cc8a5347e9b', 'admin@foo.bar', 1, '""'::json, 1), +('0a78d681-284e-4898-a80c-271edad7b1d3', 'admin@foo.bar', 1, '""'::json, 1), +('0cf2846c-a2cc-4b4c-bc53-b78bab20d678', 'admin@foo.bar', 1, '""'::json, 1), +('43ad2854-6f47-43ad-8b8f-4e77591598c0', 'admin@foo.bar', 1, '""'::json, 1), +('fc591b9e-f6df-457c-ac61-d3ddc8a97df6', 'admin@foo.bar', 1, '""'::json, 1), +('3b0ee8e6-d921-4c11-a19a-e482abff09da', 'admin@foo.bar', 1, '""'::json, 1), +('efda7c49-2865-4bf9-847a-46ec0ed48c9e', 'admin@foo.bar', 1, '""'::json, 1), +('1ef05ca4-74d6-4cb1-a0dd-64412a5e8f26', 'admin@foo.bar', 1, '""'::json, 1), +('f8a29cc6-a0cb-458e-938e-b30635035f74', 'admin@foo.bar', 1, '""'::json, 1), +('69187ffb-8155-4c06-baa9-69b3c53f901e', 'admin@foo.bar', 1, '""'::json, 1), +('d3d1f2f0-6462-453c-b346-21ae50813c3b', 'admin@foo.bar', 1, '""'::json, 1), +('f0b99e31-aa9d-4111-b2a5-0ddcb7a2c063', 'admin@foo.bar', 1, '""'::json, 1), +('449d5f69-50db-4c11-850e-ae578bdc68ad', 'admin@foo.bar', 1, '""'::json, 1), +('108c14b7-9be9-4107-be16-4e70ae0bdb8d', 'admin@foo.bar', 1, '""'::json, 1), +('3161c5b5-05c9-4513-bbc1-9b7cfbac6275', 'admin@foo.bar', 1, '""'::json, 1), +('69fcd88b-6edf-4cde-a610-0363effd0097', 'admin@foo.bar', 1, '""'::json, 1), +('a9b8ad78-8f0d-4352-aa0d-c9fb4eab8c2a', 'admin@foo.bar', 1, '""'::json, 1), +('619fa191-ec16-46f2-86bd-6d8ef583db5f', 'admin@foo.bar', 1, '""'::json, 1), +('6779287f-41eb-4733-977e-8bd8e3991a24', 'admin@foo.bar', 1, '""'::json, 1), +('ba770cd8-8667-40c9-b97e-c714cda2f80c', 'admin@foo.bar', 1, '""'::json, 1), +('58f871c7-830c-4d71-9300-e3779e203c5d', 'admin@foo.bar', 1, '""'::json, 1), +('15b23bef-7bdb-4361-891e-c8a259f22230', 'admin@foo.bar', 1, '""'::json, 1), +('246ebe66-68e4-4a9f-9a97-06f881c83cbd', 'admin@foo.bar', 1, '""'::json, 1), +('99529852-6a5c-4d5b-b029-3455aa3a8c8d', 'admin@foo.bar', 1, '""'::json, 1), +('1a8cffdc-13d8-4779-a38f-a9abeb854e44', 'admin@foo.bar', 1, '""'::json, 1), +('144380ec-debc-468a-9f7a-930197d9547b', 'admin@foo.bar', 1, '""'::json, 1), +('6365416f-e395-45d2-b558-270d0992fa0f', 'admin@foo.bar', 1, '""'::json, 1), +('eeb06d17-12d1-45c1-ac8d-4e7473983ebf', 'admin@foo.bar', 1, '""'::json, 1), +('4c38dfa2-b27b-4409-ba90-2c75eea4bba4', 'admin@foo.bar', 1, '""'::json, 1), +('a70578ed-87e9-4ef7-a1d2-760d76962e40', 'admin@foo.bar', 1, '""'::json, 1), +('eaffc15a-0d71-427b-b672-0487943d7da7', 'admin@foo.bar', 1, '""'::json, 1), +('c86955d9-980c-46be-af88-26d4ac2d2a7d', 'admin@foo.bar', 1, '""'::json, 1), +('7f7fd4fe-d7df-4e7d-a37c-936bc8377f5d', 'admin@foo.bar', 1, '""'::json, 1), +('ba61cf3b-6c70-456d-8e9e-c8541a2a6407', 'admin@foo.bar', 1, '""'::json, 1), +('ed986358-8a50-4b19-af8b-d5df11595183', 'admin@foo.bar', 1, '""'::json, 1), +('70f9bec4-9459-46f5-aeab-ac3ac7f55cf1', 'admin@foo.bar', 1, '""'::json, 1), +('38c42216-080e-413c-8c79-f3a27ab556bc', 'admin@foo.bar', 1, '""'::json, 1), +('d3c8f668-6393-4e41-8160-a17e516ff500', 'admin@foo.bar', 1, '""'::json, 1), +('692c4d20-72b3-4627-8dad-7379174604a1', 'admin@foo.bar', 1, '""'::json, 1), +('3382c6a9-b778-4a75-9092-6c76afeeb45e', 'admin@foo.bar', 1, '""'::json, 1), +('bbfa6b71-fd64-45a3-a66e-5d0d7f454e9d', 'admin@foo.bar', 1, '""'::json, 1), +('27b552f9-b3be-4fbf-93dc-323774590e1b', 'admin@foo.bar', 1, '""'::json, 1), +('4af523ba-0276-48a3-bfac-246ff361310b', 'admin@foo.bar', 1, '""'::json, 1), +('66630aa4-c73b-4dbd-9290-b5a6ff8d0129', 'admin@foo.bar', 1, '""'::json, 1), +('9e5aa6a4-23f4-4949-82df-748e4a651d62', 'admin@foo.bar', 1, '""'::json, 1), +('b6965426-898a-46e1-a40f-2fe34a9d7bb1', 'admin@foo.bar', 1, '""'::json, 1), +('052163a0-2807-40fc-8ae7-ecb8c4c1d421', 'admin@foo.bar', 1, '""'::json, 1), +('bd0b608a-05cd-4c67-a4d9-e3337075e2e4', 'admin@foo.bar', 1, '""'::json, 1), +('a5d9bb39-116b-49e3-97c4-831580f6729f', 'admin@foo.bar', 1, '""'::json, 1), +('cefcf3fb-26f2-4349-9fc8-d81c63e928ff', 'admin@foo.bar', 1, '""'::json, 1), +('c573d2c7-91bd-4dfe-a433-a6ef89a9656c', 'admin@foo.bar', 1, '""'::json, 1), +('869f9610-e808-4935-a57e-781abc9960af', 'admin@foo.bar', 1, '""'::json, 1), +('83c880c1-20e6-4134-aca4-ced663066543', 'admin@foo.bar', 1, '""'::json, 1), +('cd376889-4d87-40f8-8d3e-b9a4c0623f55', 'admin@foo.bar', 1, '""'::json, 1), +('d8d739f2-6d56-41b2-a5a7-38133f9d7bbd', 'admin@foo.bar', 1, '""'::json, 1), +('be224c35-461b-48a3-ab48-4e4838019c64', 'admin@foo.bar', 1, '""'::json, 1), +('8b99435f-643a-4347-b552-18a5aea558e0', 'admin@foo.bar', 1, '""'::json, 1), +('9d372971-7763-4731-a44e-cb87a3434cc0', 'admin@foo.bar', 1, '""'::json, 1), +('9a3a5d72-c2b0-449a-a157-1c088e0af56d', 'admin@foo.bar', 1, '""'::json, 1), +('a8afae6b-c328-48df-bce0-bab522a98f85', 'admin@foo.bar', 1, '""'::json, 1), +('1fa748d3-fa80-4af4-a8fa-7782a27fa585', 'admin@foo.bar', 1, '""'::json, 1), +('559da3f1-bc29-4c26-a6ac-06b52b054594', 'admin@foo.bar', 1, '""'::json, 1), +('cb41a3f9-0be7-4dad-b015-68016ef51022', 'admin@foo.bar', 1, '""'::json, 1), +('19203be8-1052-4c80-9bce-fff4e77f05c8', 'admin@foo.bar', 1, '""'::json, 1), +('a91d644d-9c86-411c-9ae2-4eeac7545d2b', 'admin@foo.bar', 1, '""'::json, 1), +('5e0bba68-4bfb-40e1-96d1-daf1a454fd46', 'admin@foo.bar', 1, '""'::json, 1), +('510e59c2-ba8e-4b1c-b4be-17dc59c80457', 'admin@foo.bar', 1, '""'::json, 1), +('a4b6745c-bad5-470a-8e32-2f75236c2fce', 'admin@foo.bar', 1, '""'::json, 1), +('f69d7684-607e-4661-a836-c09aba157c95', 'admin@foo.bar', 1, '""'::json, 1), +('5c679aad-68f7-4855-be14-8312110bc173', 'admin@foo.bar', 1, '""'::json, 1), +('e015a0a1-5c4f-4178-8ebc-2b9ac9e4e8b6', 'admin@foo.bar', 1, '""'::json, 1), +('cd2374bc-649c-45e2-8540-991fb36cb487', 'admin@foo.bar', 1, '""'::json, 1), +('7c707855-1640-42e9-82a7-d720e58874eb', 'admin@foo.bar', 1, '""'::json, 1), +('20af2ade-f786-4305-85b1-b5272ab356e0', 'admin@foo.bar', 1, '""'::json, 1), +('490b3500-dc03-4fa9-9cd2-78126b9333c2', 'admin@foo.bar', 1, '""'::json, 1), +('e5504565-cf0e-4f2d-b925-1c2ca7cb5fbb', 'admin@foo.bar', 1, '""'::json, 1), +('f9a9e2d7-b568-45d6-89b8-89cfe9991f77', 'admin@foo.bar', 1, '""'::json, 1), +('8500b61a-9f08-49c9-a897-2773f4f383c5', 'admin@foo.bar', 1, '""'::json, 1), +('7e8480aa-af3b-4df7-8d66-8f93a28ccb74', 'admin@foo.bar', 1, '""'::json, 1), +('f87a89b9-4b6e-4398-8e48-0b6ec238cc97', 'admin@foo.bar', 1, '""'::json, 1), +('92b6145c-1248-4024-b825-a8b6cb07c4b0', 'admin@foo.bar', 1, '""'::json, 1), +('04e67d15-1907-4fd1-9a4d-48b47f6c71f8', 'admin@foo.bar', 1, '""'::json, 1), +('f7466925-9a5c-40d8-9536-9c43ad30cc2d', 'admin@foo.bar', 1, '""'::json, 1), +('2f21b2ec-91b2-43e2-8d70-b35b64fcaf0c', 'admin@foo.bar', 1, '""'::json, 1), +('3df051d5-99a8-411a-b40f-bc269eeb5a6c', 'admin@foo.bar', 1, '""'::json, 1), +('6048e851-42a4-4b0b-b040-a05cfe9686b0', 'admin@foo.bar', 1, '""'::json, 1), +('06bbba17-3761-4e80-a7c0-56a01e92e7f2', 'admin@foo.bar', 1, '""'::json, 1), +('1d1f338a-0b0b-4597-a960-1ecf5019dcfe', 'admin@foo.bar', 1, '""'::json, 1), +('ba1c6fdd-bedd-45f1-a255-aa6b5e21fbd7', 'admin@foo.bar', 1, '""'::json, 1), +('1f0a1ce3-35b9-427a-b11d-096690fe4a27', 'admin@foo.bar', 1, '""'::json, 1), +('194ec434-bedd-4403-9e14-2351ea3c7720', 'admin@foo.bar', 1, '""'::json, 1), +('633b6978-0c6b-400a-b466-f3eb7e2035e9', 'admin@foo.bar', 1, '""'::json, 1), +('dd18384b-c038-48bf-b751-573163a4223d', 'admin@foo.bar', 1, '""'::json, 1), +('635c4f2a-8417-4fae-9078-e94846c8b7b1', 'admin@foo.bar', 1, '""'::json, 1), +('759be360-787b-410e-9e07-7db03e4df5c6', 'admin@foo.bar', 1, '""'::json, 1), +('37d514e8-ce44-4163-b47b-f0af05397aff', 'admin@foo.bar', 1, '""'::json, 1), +('7b0f4fc2-d772-4921-ad44-ba64f148a84d', 'admin@foo.bar', 1, '""'::json, 1), +('9a730dc2-23b7-4c41-8270-5fa7af22d468', 'admin@foo.bar', 1, '""'::json, 1), +('c033bd49-d4e9-4df3-9317-ce030d4b80d6', 'admin@foo.bar', 1, '""'::json, 1), +('09ee747d-638c-4933-bbe6-4e999d4b7480', 'admin@foo.bar', 1, '""'::json, 1), +('16060ef4-c199-493f-ab88-d4b4f06138a4', 'admin@foo.bar', 1, '""'::json, 1), +('8c1f6f46-75e6-4439-a5c0-c4e9bad71ec0', 'admin@foo.bar', 1, '""'::json, 1), +('22c6824e-2223-4a56-8686-dcbf3db8b2f8', 'admin@foo.bar', 1, '""'::json, 1), +('45c822ad-7817-48b6-8208-7b65922a5b01', 'admin@foo.bar', 1, '""'::json, 1), +('40e38e61-5426-4495-95a8-9bfd717b779e', 'admin@foo.bar', 1, '""'::json, 1), +('6b7f13b9-d019-4cd7-9913-6cc2a2d558c0', 'admin@foo.bar', 1, '""'::json, 1), +('643a6bdc-48a3-4624-ab93-1e70053c0f06', 'admin@foo.bar', 1, '""'::json, 1), +('d97b7b26-6c93-4d55-bd52-c4beae6fd9fc', 'admin@foo.bar', 1, '""'::json, 1), +('caab8faa-5ebd-4734-91d3-cc598fe332c4', 'admin@foo.bar', 1, '""'::json, 1), +('a005ae56-c6a3-4e0f-8280-49d85e32023e', 'admin@foo.bar', 1, '""'::json, 1), +('1a93bbf7-b118-4b67-9477-e5a067291f7e', 'admin@foo.bar', 1, '""'::json, 1), +('670fcf04-077d-41cd-b9b6-d5eb7b37b42a', 'admin@foo.bar', 1, '""'::json, 1), +('97704373-2097-4e0f-aa15-5a6087877b86', 'admin@foo.bar', 1, '""'::json, 1), +('3f1f3955-04a0-4554-ad87-55b4c2b13ee4', 'admin@foo.bar', 1, '""'::json, 1), +('97d9b7c0-b68d-41ff-954c-3755e6ff1cb9', 'admin@foo.bar', 1, '""'::json, 1), +('15f391a5-8c92-46e0-a321-5b981848d586', 'admin@foo.bar', 1, '""'::json, 1), +('e0d0e1ad-55f7-4486-a53a-7df8b44bd4ab', 'admin@foo.bar', 1, '""'::json, 1), +('b5415610-17a1-4ac7-a985-a060df556451', 'admin@foo.bar', 1, '""'::json, 1), +('80dc4574-8278-4492-9473-f6091c52b98c', 'admin@foo.bar', 1, '""'::json, 1), +('f838c5ba-bc20-4c5c-94cb-52973b522be4', 'admin@foo.bar', 1, '""'::json, 1), +('5a1d5e55-4fe4-4dba-9406-684ffb734c5f', 'admin@foo.bar', 1, '""'::json, 1), +('a5f23b08-8499-4e16-9b6c-69d089c9fd09', 'admin@foo.bar', 1, '""'::json, 1), +('2b365ffc-a96b-47a6-89c2-6d2aaae535df', 'admin@foo.bar', 1, '""'::json, 1), +('32a9bb7f-56c2-4bbc-9120-d66d4d8debbc', 'admin@foo.bar', 1, '""'::json, 1), +('2a1d211e-5b53-4121-93f6-49721523eda4', 'admin@foo.bar', 1, '""'::json, 1), +('bffc8bce-1f6f-4bff-ac7f-8dad7f43895a', 'admin@foo.bar', 1, '""'::json, 1), +('fc86707d-54ca-4eee-92d3-dbcde06ed75c', 'admin@foo.bar', 1, '""'::json, 1), +('0c7ddae6-83d6-4db6-9a79-9720c614142b', 'admin@foo.bar', 1, '""'::json, 1), +('9194b7f9-a420-426e-8073-78ad6607cd1d', 'admin@foo.bar', 1, '""'::json, 1), +('db9b1f49-f848-4a40-bbbb-ed9d3848965a', 'admin@foo.bar', 1, '""'::json, 1), +('84eff552-5723-4729-a86b-8f283874c289', 'admin@foo.bar', 1, '""'::json, 1), +('b251dfae-0bb8-4bee-ab47-bdcc17d25b62', 'admin@foo.bar', 1, '""'::json, 1), +('316b9754-2d5d-4822-bd72-f86f6ca9ae4b', 'admin@foo.bar', 1, '""'::json, 1), +('d509e593-4053-406a-8ab0-8af8b1b4c8a5', 'admin@foo.bar', 1, '""'::json, 1), +('e66faf00-0ffa-404f-ac08-7121c6337b70', 'admin@foo.bar', 1, '""'::json, 1), +('66782585-883b-4784-a443-a2dfff848d88', 'admin@foo.bar', 1, '""'::json, 1), +('9291cc5a-9ab4-49c0-b8a1-9fc9b6c46b44', 'admin@foo.bar', 1, '""'::json, 1), +('decafc1b-5c70-466b-a8cc-33f29c6db3df', 'admin@foo.bar', 1, '""'::json, 1), +('5b319607-4bec-4e36-ae54-5b7c21e91064', 'admin@foo.bar', 1, '""'::json, 1), +('2f3f309f-51bd-43b8-80d1-10d10a1c274e', 'admin@foo.bar', 1, '""'::json, 1), +('893621a8-d4dd-464b-8977-223470fc297e', 'admin@foo.bar', 1, '""'::json, 1), +('12e91952-b2b1-424d-884c-865e77474bf2', 'admin@foo.bar', 1, '""'::json, 1), +('d6946499-e2e7-4286-a95b-786b8ec633a9', 'admin@foo.bar', 1, '""'::json, 1), +('1fa39acc-83ef-409f-91d5-13ef1934635d', 'admin@foo.bar', 1, '""'::json, 1), +('694a13a5-9e5d-4f54-b34c-a63167a045b1', 'admin@foo.bar', 1, '""'::json, 1), +('c4ff14f9-3a89-40d6-aa85-1fbd8dd80c17', 'admin@foo.bar', 1, '""'::json, 1), +('d12d6d89-a8e9-4fd9-8e1c-16968035fb38', 'admin@foo.bar', 1, '""'::json, 1), +('9627e0a4-8f47-430e-b1d2-30c1295a722f', 'admin@foo.bar', 1, '""'::json, 1), +('f50accb5-3461-4a7f-9bff-761e4414a026', 'admin@foo.bar', 1, '""'::json, 1), +('e0d4954f-e964-4755-bac3-bc47610206aa', 'admin@foo.bar', 1, '""'::json, 1), +('2d60289d-1b7f-4945-8264-b01e2426c23e', 'admin@foo.bar', 1, '""'::json, 1), +('179d0c15-5af2-4671-b807-fb374fd3b842', 'admin@foo.bar', 1, '""'::json, 1), +('f6a8189a-caa7-4a41-bbc0-09ac46bf5a8f', 'admin@foo.bar', 1, '""'::json, 1), +('59ca6817-c388-47ad-ba64-47989313dd44', 'admin@foo.bar', 1, '""'::json, 1), +('61d6d576-362c-4261-859e-7209fa346f1e', 'admin@foo.bar', 1, '""'::json, 1), +('bd89bfe7-51b0-429b-9a88-2e71c8cb7545', 'admin@foo.bar', 1, '""'::json, 1), +('c5cc0f29-0fa0-4c44-b33c-103f0e3b6f32', 'admin@foo.bar', 1, '""'::json, 1), +('53db452b-9234-4830-94ef-cf9ab49ef0d1', 'admin@foo.bar', 1, '""'::json, 1), +('a268c541-fe8d-4521-8a49-e3a8439aef64', 'admin@foo.bar', 1, '""'::json, 1), +('387f71b9-3283-4923-860b-8065a62248fd', 'admin@foo.bar', 1, '""'::json, 1), +('8194c64a-1a43-499f-8f47-3a3935c557f4', 'admin@foo.bar', 1, '""'::json, 1), +('294d4a3f-f8ff-4644-ab15-3ddcb4390af1', 'admin@foo.bar', 1, '""'::json, 1), +('a373adcd-9b62-41e3-9ae4-a6f79ff45f45', 'admin@foo.bar', 1, '""'::json, 1), +('ef7406a1-a4fa-4af2-8e67-92607ebdd8f9', 'admin@foo.bar', 1, '""'::json, 1), +('59dbbe08-8701-4e48-95ab-3882a02ee1e7', 'admin@foo.bar', 1, '""'::json, 1), +('088a65e5-c848-4e34-b47d-4664f9f27f83', 'admin@foo.bar', 1, '""'::json, 1), +('45d538e2-97c5-4893-af08-f6bf04abb767', 'admin@foo.bar', 1, '""'::json, 1), +('0b9706a2-e0a6-438f-a5b3-a094161a15a7', 'admin@foo.bar', 1, '""'::json, 1), +('eb3650f0-7aea-4cab-a520-6e581e3bd978', 'admin@foo.bar', 1, '""'::json, 1), +('c6635e7e-93c0-403e-b615-3e8f7da29352', 'admin@foo.bar', 1, '""'::json, 1), +('2cb50802-1ac7-48a4-bebd-4d9be546cf69', 'admin@foo.bar', 1, '""'::json, 1), +('1c6d9feb-17fb-40ca-a3f4-d1e7834c5d53', 'admin@foo.bar', 1, '""'::json, 1), +('ed2a60cf-2164-4b01-a59c-07518613ef8d', 'admin@foo.bar', 1, '""'::json, 1), +('469e8fcc-37ef-4dfb-b4cd-2b109200ce4c', 'admin@foo.bar', 1, '""'::json, 1), +('52802b61-7df2-4e82-9ccf-5c8075c2da45', 'admin@foo.bar', 1, '""'::json, 1), +('0c715e72-47dd-48ac-b733-a845e440e86a', 'admin@foo.bar', 1, '""'::json, 1), +('13f6b03d-40e1-47c2-a7df-163c0e3e7d23', 'admin@foo.bar', 1, '""'::json, 1), +('9639186c-9084-47ed-8075-3f14798eceef', 'admin@foo.bar', 1, '""'::json, 1), +('6e54aa84-ffd1-4943-8ae2-7e75319a7e30', 'admin@foo.bar', 1, '""'::json, 1), +('dba3182c-80c0-46ed-a9e6-64e0cf699741', 'admin@foo.bar', 1, '""'::json, 1), +('b33244f5-30ce-4db3-94c7-4dcbe5cff4a7', 'admin@foo.bar', 1, '""'::json, 1), +('ba196947-c2d6-4ae1-ad23-fb768c087125', 'admin@foo.bar', 1, '""'::json, 1), +('08d92acd-992c-4500-a33a-094225dec9cb', 'admin@foo.bar', 1, '""'::json, 1), +('b0bfe252-9f89-4160-839e-738a649de0f1', 'admin@foo.bar', 1, '""'::json, 1), +('ea9e5e73-c141-4cf6-944a-6f43541ed747', 'admin@foo.bar', 1, '""'::json, 1), +('b6d61911-54b5-4917-957a-11f5931f1b6e', 'admin@foo.bar', 1, '""'::json, 1), +('3cf83989-1316-47f4-a291-03aa41da31f1', 'admin@foo.bar', 1, '""'::json, 1), +('531b6413-c293-4a87-9444-3939eebf79bd', 'admin@foo.bar', 1, '""'::json, 1), +('18919546-fade-4e83-b5dc-1708043bec82', 'admin@foo.bar', 1, '""'::json, 1), +('6e6eb332-9166-4059-8a7c-cc4dfa5417f3', 'admin@foo.bar', 1, '""'::json, 1), +('c36b0187-97f8-49c4-9f78-dfb75a04f41d', 'admin@foo.bar', 1, '""'::json, 1), +('f9552ba6-db05-4a74-acb7-f851c627777d', 'admin@foo.bar', 1, '""'::json, 1), +('4777ec99-b163-466f-81a8-a5e5f3c81b5d', 'admin@foo.bar', 1, '""'::json, 1), +('3429c0f3-9f05-4234-a3ec-ead9285d9c91', 'admin@foo.bar', 1, '""'::json, 1), +('a9e4fa1b-5f48-4143-a06e-20c6013c37f0', 'admin@foo.bar', 1, '""'::json, 1), +('eb734567-fa4f-4aff-8ac4-e119b7cf31cc', 'admin@foo.bar', 1, '""'::json, 1), +('52b3c3ce-0681-4992-9cd5-bdb0eddffff6', 'admin@foo.bar', 1, '""'::json, 1), +('ddb6e8c1-700b-4c2e-834b-26ceca5be035', 'admin@foo.bar', 1, '""'::json, 1), +('ed55d400-04c2-4aa7-be76-63eef565cbae', 'admin@foo.bar', 1, '""'::json, 1), +('23d23e6a-7e1e-41e6-aedb-5c997c6a31a6', 'admin@foo.bar', 1, '""'::json, 1), +('160160e6-ee83-47f7-8d59-1e6df6dd1a4d', 'admin@foo.bar', 1, '""'::json, 1), +('e3b07a6e-145b-4551-b27c-905a1601a1f3', 'admin@foo.bar', 1, '""'::json, 1), +('462b169a-acce-47a7-adf9-1daa0a465639', 'admin@foo.bar', 1, '""'::json, 1), +('a43beee6-aab4-4ddd-895e-b7f95e258d17', 'admin@foo.bar', 1, '""'::json, 1), +('4df02286-2ecd-4dd0-9d13-ec5d6b2e76d9', 'admin@foo.bar', 1, '""'::json, 1), +('50b68208-0bdc-4ecd-9f75-2036a2332d55', 'admin@foo.bar', 1, '""'::json, 1), +('9bba65d6-12be-4f4b-b40a-018489ebbc3b', 'admin@foo.bar', 1, '""'::json, 1), +('40fe74d9-7fa2-4ef1-a64d-c092b3e5128b', 'admin@foo.bar', 1, '""'::json, 1), +('716b59bd-d2f0-404e-b6c5-ad5b69e8db3a', 'admin@foo.bar', 1, '""'::json, 1), +('cd5299dd-f160-4b8e-9eac-ea2cb54c003a', 'admin@foo.bar', 1, '""'::json, 1), +('6140d69a-3c93-4378-bc99-fc0596aa6703', 'admin@foo.bar', 1, '""'::json, 1), +('c03cd275-e2c0-4e28-a0a1-e07be2c24c03', 'admin@foo.bar', 1, '""'::json, 1), +('9afa1b4a-244e-44a4-8b9c-69c229007667', 'admin@foo.bar', 1, '""'::json, 1), +('655e4674-a14d-485b-9294-18d8e76014ca', 'admin@foo.bar', 1, '""'::json, 1), +('2a009904-eb9e-49bd-bee0-da0bb1bfdba5', 'admin@foo.bar', 1, '""'::json, 1), +('186f76c0-c877-428c-ab08-46cbcf4524a1', 'admin@foo.bar', 1, '""'::json, 1), +('f5d2cdc0-0061-4a7b-9c86-bee6d7e08d29', 'admin@foo.bar', 1, '""'::json, 1), +('f147fe13-56a8-41db-b399-ecabba242ca9', 'admin@foo.bar', 1, '""'::json, 1), +('0bef5c46-6ad7-4a05-8c4b-8d227f0a0352', 'admin@foo.bar', 1, '""'::json, 1), +('fc0ebaac-1bbb-4fb6-b20d-2fe74f31b5ef', 'admin@foo.bar', 1, '""'::json, 1), +('fcd7e83f-804c-40bb-9f3d-8483cffe34c0', 'admin@foo.bar', 1, '""'::json, 1), +('284cc76f-487e-4d49-9c2f-d4d10e74385e', 'admin@foo.bar', 1, '""'::json, 1), +('b8e3c3ac-5b5b-4f6d-97c2-3296b0bab7e6', 'admin@foo.bar', 1, '""'::json, 1), +('9262c5ec-b92b-425b-8fd1-10ea4905d595', 'admin@foo.bar', 1, '""'::json, 1), +('7bae4416-a699-4985-9736-60bf2b4aa4d6', 'admin@foo.bar', 1, '""'::json, 1), +('7cb7e7e1-48c1-4259-9856-395911e88d4d', 'admin@foo.bar', 1, '""'::json, 1), +('b127f3dc-fc5b-4054-a20b-e9216caff4ef', 'admin@foo.bar', 1, '""'::json, 1), +('dfb12c24-d67a-4dc4-b221-b6fb1e49602b', 'admin@foo.bar', 1, '""'::json, 1), +('507da6a3-0df4-4e30-9763-f392eb62aac5', 'admin@foo.bar', 1, '""'::json, 1), +('6fe5c170-4774-4576-813f-7c164866a3bf', 'admin@foo.bar', 1, '""'::json, 1), +('3c31aad8-1e86-48be-b65c-4924388bd587', 'admin@foo.bar', 1, '""'::json, 1), +('2b2e2473-fbc6-4e6a-9072-19f6411b0817', 'admin@foo.bar', 1, '""'::json, 1), +('5994ad72-2f1a-416b-ba3c-9cc4a1c61c20', 'admin@foo.bar', 1, '""'::json, 1), +('ba111653-1c15-4425-8ab9-11a5422eff1d', 'admin@foo.bar', 1, '""'::json, 1), +('fdec2e67-e193-4185-991a-be136e63b2b0', 'admin@foo.bar', 1, '""'::json, 1), +('6218ce6d-5208-4ebc-9e48-633b3f54cdcc', 'admin@foo.bar', 1, '""'::json, 1), +('d6eef75a-4505-4111-9c4d-5a748d403dd2', 'admin@foo.bar', 1, '""'::json, 1), +('7a24dba0-f6b4-48cc-b1cb-4c9b884170e6', 'admin@foo.bar', 1, '""'::json, 1), +('ff077c77-a1ba-46ac-b5ca-d5c894242770', 'admin@foo.bar', 1, '""'::json, 1), +('65526acd-86e9-421b-9c91-b885cd36729a', 'admin@foo.bar', 1, '""'::json, 1), +('04d09a9f-2151-47df-8475-36fc8ec7aafc', 'admin@foo.bar', 1, '""'::json, 1), +('7e72e913-ce87-4bc1-adf7-c3cc7d5184e5', 'admin@foo.bar', 1, '""'::json, 1), +('9e6f2ff5-f749-4089-b828-ef29fcca1a7d', 'admin@foo.bar', 1, '""'::json, 1), +('335635a7-8153-4e4c-9b53-e1bc145497b0', 'admin@foo.bar', 1, '""'::json, 1), +('5452c17e-ae82-4b46-a85a-ece0501a5d5e', 'admin@foo.bar', 1, '""'::json, 1), +('f5a06f01-7405-4ff1-b047-64f9e7505d6c', 'admin@foo.bar', 1, '""'::json, 1), +('bf9fc8f3-39b7-4771-95e8-f858494022a5', 'admin@foo.bar', 1, '""'::json, 1), +('98270055-d47a-4cc4-b40c-ac006f7823f6', 'admin@foo.bar', 1, '""'::json, 1), +('009b95ec-18b6-4fa0-a0bb-0785cfe03409', 'admin@foo.bar', 1, '""'::json, 1), +('89ae9ba5-f920-46e6-9279-40d5be29f845', 'admin@foo.bar', 1, '""'::json, 1), +('0f24f638-13fe-4898-b3a9-b15aafb260f8', 'admin@foo.bar', 1, '""'::json, 1), +('37d12cdd-c03b-4921-a77b-851269d3e557', 'admin@foo.bar', 1, '""'::json, 1), +('9308c777-1c30-43ac-acf6-9454601601ba', 'admin@foo.bar', 1, '""'::json, 1), +('a1910b9b-2d93-4239-9bc1-1250fea56837', 'admin@foo.bar', 1, '""'::json, 1), +('1280dc5a-d797-42ca-a613-3fbce32f55a0', 'admin@foo.bar', 1, '""'::json, 1), +('0a329ce1-1b82-4faf-a1f7-0d17e4b25251', 'admin@foo.bar', 1, '""'::json, 1), +('3b212797-bb0b-422c-bba1-79921ce40c78', 'admin@foo.bar', 1, '""'::json, 1), +('c14e97a2-a083-4e49-a14b-0abd23483272', 'admin@foo.bar', 1, '""'::json, 1), +('7cd84e00-773a-4387-bcdb-37da819ce5f7', 'admin@foo.bar', 1, '""'::json, 1), +('8553c8e2-5bc9-41b6-9f5e-96a2abfb7055', 'admin@foo.bar', 1, '""'::json, 1), +('65212822-529e-4cf4-a427-78032b9e569b', 'admin@foo.bar', 1, '""'::json, 1), +('3a8d8b29-d408-45ab-8d63-043741de9679', 'admin@foo.bar', 1, '""'::json, 1), +('d68f1265-c669-4f2b-a3ab-ee0e547e0143', 'admin@foo.bar', 1, '""'::json, 1), +('f0b0c5aa-9f97-42e3-82e4-71968f28ad15', 'admin@foo.bar', 1, '""'::json, 1), +('1bf64e7e-4ab9-4dfa-9aeb-6402ad95c165', 'admin@foo.bar', 1, '""'::json, 1), +('605f6954-8697-42c4-97a4-7c3aa6b3cde9', 'admin@foo.bar', 1, '""'::json, 1), +('3688f502-6895-4041-b957-6487f6ba7c98', 'admin@foo.bar', 1, '""'::json, 1), +('013f8041-6f36-4dcb-8af9-67ac1fd78d9e', 'admin@foo.bar', 1, '""'::json, 1), +('58e69a18-ecef-4b61-a62f-7efc1f733731', 'admin@foo.bar', 1, '""'::json, 1), +('e7cb303d-ffe0-4a5e-8eb0-627dd0513f51', 'admin@foo.bar', 1, '""'::json, 1), +('7cc4a810-137c-468e-a249-60b0faf9a003', 'admin@foo.bar', 1, '""'::json, 1), +('056330e4-36eb-4c38-b87f-56484ac69623', 'admin@foo.bar', 1, '""'::json, 1), +('ea9e5c11-f7a7-4dcb-b8b5-20b499611b9f', 'admin@foo.bar', 1, '""'::json, 1), +('8df2b291-adbf-4def-a172-ece656410050', 'admin@foo.bar', 1, '""'::json, 1), +('ce47b98d-8a78-4e97-b20c-297f75394f28', 'admin@foo.bar', 1, '""'::json, 1), +('d8e90f5f-1b10-4f00-9c3b-6bc8daf2cc8a', 'admin@foo.bar', 1, '""'::json, 1), +('cec46141-0286-4430-ac41-ed830decb5e8', 'admin@foo.bar', 1, '""'::json, 1), +('fc9b54c3-0447-49db-a6d0-5b913d0e285a', 'admin@foo.bar', 1, '""'::json, 1), +('3e6d362e-3880-4b90-b693-868547ce0f5a', 'admin@foo.bar', 1, '""'::json, 1), +('d5891efa-0e7d-409b-9bb7-b6ecc081c836', 'admin@foo.bar', 1, '""'::json, 1), +('e4b960c8-33c6-4304-8355-9bafd70569ed', 'admin@foo.bar', 1, '""'::json, 1), +('35c4eb46-3560-4d17-a210-979ba3037c53', 'admin@foo.bar', 1, '""'::json, 1), +('2b85bd17-6d7e-48e0-8e02-477eb9ffe353', 'admin@foo.bar', 1, '""'::json, 1), +('8c7e4562-dcc2-4a31-ab20-b873182b334d', 'admin@foo.bar', 1, '""'::json, 1), +('49fda716-a469-4a6d-a3ef-3bcbd166e668', 'admin@foo.bar', 1, '""'::json, 1), +('f424b26a-840a-42d3-a61f-3bbffd8b194d', 'admin@foo.bar', 1, '""'::json, 1), +('7226d658-da67-4d7c-a898-e59cd77d977b', 'admin@foo.bar', 1, '""'::json, 1), +('50dc9288-e18c-41c2-a271-601289ff5685', 'admin@foo.bar', 1, '""'::json, 1), +('4883db35-3ca6-4b0c-9377-cf7d1d97245d', 'admin@foo.bar', 1, '""'::json, 1), +('3ff09f4e-ada5-45d6-b7e1-fa003e31bf77', 'admin@foo.bar', 1, '""'::json, 1), +('26c68022-6df2-4928-a4fb-220a156b86f7', 'admin@foo.bar', 1, '""'::json, 1), +('cdd878a9-e3f1-4478-b613-90f2b2e9ccbd', 'admin@foo.bar', 1, '""'::json, 1), +('d6e02749-45fb-4b92-8a8e-48b33b617f9c', 'admin@foo.bar', 1, '""'::json, 1), +('47900ec7-0fea-41db-a320-c78b9da36538', 'admin@foo.bar', 1, '""'::json, 1), +('1ef4d550-afaa-44b5-a06c-65286b9f3e00', 'admin@foo.bar', 1, '""'::json, 1), +('8533d2a8-1fcf-4842-8826-c5458c002332', 'admin@foo.bar', 1, '""'::json, 1), +('ab9095d1-af6d-49d5-9c87-bc2576f06135', 'admin@foo.bar', 1, '""'::json, 1), +('61da73ff-b4ff-49a1-b775-c6215cfbd291', 'admin@foo.bar', 1, '""'::json, 1), +('6c84dcf1-c5ea-4e69-b17f-d2d5b8d48bdf', 'admin@foo.bar', 1, '""'::json, 1), +('dcb12603-4142-44d1-9a52-3ca3511e380e', 'admin@foo.bar', 1, '""'::json, 1), +('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 'admin@foo.bar', 1, '""'::json, 1),; +-- SPLIT + +INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, columns, input_size, extra_info, memory_used, walltime_used) +VALUES ('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', 327036000, 91), +('b0f36550-d97c-4dd5-ba1b-910470062801', 160, 212, 2, 'nan', 342204000, 219), +('b13eefd1-12b7-4c3e-a380-94ec0a3f9f91', 62, 83, 2, 'nan', 328716000, 165), +('469c27b7-81c4-4f41-b09d-149260961227', 16, 83, 2, 'nan', 326344000, 154), +('6f198e33-b4ae-47e9-973b-bd5e0b06c353', 96, 152, 2, 'nan', 332492000, 185), +('045eb36c-5144-4d54-a621-23eb5dc4d6ef', 242, 43, 2, 'nan', 369988000, 137), +('041f3278-2752-44f9-8caa-ac6ae80c98e6', 231, 97, 2, 'nan', 375820000, 151), +('6325e683-b215-4c56-bf5a-cbbc92330067', 120, 83, 2, 'nan', 373536000, 234), +('03dcca05-0d4f-48c5-bc7a-7ef884be8efc', 92, 83, 2, 'nan', 372188000, 407), +('8d84685f-97f4-4ea7-8857-2dcec5a20775', 40, 83, 2, 'nan', 368816000, 296), +('8b714fe7-51d9-492c-9b49-4fe09b595067', 96, 87, 2, 'nan', 371972000, 138), +('490c0623-4ed7-4064-9ffe-ba5afc0f0aa0', 28, 50, 2, 'nan', 369696000, 185), +('c8301f3d-a4fe-407a-9aec-b9dfee0cc2fd', 12, 83, 2, 'nan', 368052000, 97), +('561b9676-604a-4cdb-833f-a4a9b3afef42', 41, 50, 2, 'nan', 369580000, 101), +('8bc35697-f3df-42f6-8a34-830d42000bd6', 336, 22, 2, 'nan', 369828000, 86), +('474a4a8d-0963-4fe6-9da4-74e520532be1', 32, 80, 2, 'nan', 364956000, 105), +('d4d03b6f-12f1-4f47-9ad4-b5597a3e5a16', 152, 66, 2, 'nan', 373528000, 129), +('27d95aba-9b36-40c8-8d17-7ade170172aa', 36, 83, 2, 'nan', 367776000, 155), +('a8f00090-6981-4fc3-bc37-208f01e6dceb', 44, 83, 2, 'nan', 370620000, 93), +('d7562d61-7886-45b5-9fed-90fef6636e6f', 152, 83, 2, 'nan', 372888000, 100), +('e4071da6-550d-469b-91dc-f6cda658df80', 32, 64, 2, 'nan', 369900000, 155), +('c126e507-5f13-485f-9166-775ee7228f12', 28, 26, 2, 'nan', 368464000, 97), +('d73906a4-f2c7-4d03-bd55-d637365ec3bc', 138, 89, 2, 'nan', 375516000, 288), +('167efba9-a88e-493c-a2c6-0647556ba6dd', 35, 42, 2, 'nan', 369480000, 81), +('2662c176-d042-4254-bc7e-20c092fc3af1', 152, 72, 2, 'nan', 373376000, 135), +('54b19d0d-fa47-4c37-a4ea-e2be33d54c5b', 152, 80, 2, 'nan', 369868000, 92), +('8b3e969e-78c4-4a09-9767-4aa72d99ee8c', 90, 75, 2, 'nan', 368312000, 99), +('bef26557-60db-4987-b0bf-e9e0be77da37', 32, 83, 2, 'nan', 372748000, 100), +('adcf6cb4-780b-4a9d-bb7c-cd2bf12ac201', 28, 51, 2, 'nan', 370832000, 139), +('54464698-5860-4543-9376-d3d89722848f', 152, 81, 2, 'nan', 372652000, 95), +('9a8b3ea7-36d7-46f7-8602-1b39c89f5ee4', 28, 46, 2, 'nan', 369584000, 193), +('b57c1bc6-a805-463d-87fc-b76a186e5a63', 21302, 90, 2, 'nan', 30269928000, 187836), +('0c0579c8-dc34-4211-ad87-c453e9cb10bf', 4566, 178, 2, 'nan', 5477396000, 11596), +('04357795-299a-4900-9fdf-debf3be2ccc7', 56, 80, 2, 'nan', 370324000, 100), +('fa1617ad-daf8-496d-937f-5ad9526f57d2', 214, 84, 2, 'nan', 380764000, 139), +('056dd777-1c93-424a-ad7a-fbac4ff01e6b', 174, 81, 2, 'nan', 371052000, 105), +('1d056722-c08c-45e7-abbf-794abf85b2ba', 376, 94, 2, 'nan', 394040000, 215), +('a4e4457e-e191-4951-81d2-a1bd8afa7bb9', 189, 142, 2, 'nan', 382424000, 165), +('2bd5eedb-f4bd-4221-a31d-7a82ed1ed85c', 1067, 12, 2, 'nan', 372276000, 111), +('59e6279e-5a92-4ad5-81cd-d83892eb35e3', 6374, 49, 2, 'nan', 9936144000, 21930), +('594e5a37-e00e-44e3-8baf-a5d0fbd8d1c5', 1917, 102, 2, 'nan', 1292284000, 1367), +('40e16ce2-d280-423b-908e-f74877751bb8', 152, 85, 2, 'nan', 372464000, 159), +('df964c5d-d204-475d-bf37-c4281c830e36', 266, 67, 2, 'nan', 369540000, 98), +('f1ff0a3f-5004-49d4-a781-fedab076f682', 39, 93, 2, 'nan', 373044000, 152), +('d5b6766a-5697-4283-b7f5-ded6295e539d', 150, 63, 2, 'nan', 374136000, 444), +('6e98873c-0c99-47f0-ae6f-fc12092c1c5d', 856, 49, 2, 'nan', 584216000, 578), +('b816de4b-42f6-4495-8a05-7cb7faefcf3e', 856, 48, 2, 'nan', 634320000, 557), +('2e196649-1641-4ced-bf74-f24db6123d66', 394, 42, 2, 'nan', 380860000, 767), +('ca8271ce-ff25-4a22-8309-149396d20d38', 384, 22, 2, 'nan', 376636000, 374), +('38496999-f362-45d1-a9e6-6c2ed7ad1847', 69, 70, 2, 'nan', 373668000, 101), +('dedeab8b-ed71-4d3d-87c1-37f607e19e5d', 20, 47, 2, 'nan', 365196000, 280), +('997ad2aa-59ff-4346-805c-cbe1a41bac00', 413, 40, 2, 'nan', 412392000, 181), +('ff866805-7add-432f-90a8-15ab4b793cdd', 1005, 137, 2, 'nan', 425864000, 165), +('ef8a5faa-fc7b-4a09-a45a-7aa4d3dbb1f8', 177, 384, 2, 'nan', 380400000, 231), +('b0db6327-e4ff-4dd5-85ae-b8a060f8ed9f', 96, 99, 2, 'nan', 376124000, 234), +('900537c6-a4bb-4f20-9674-e0fd4851c10a', 22, 47, 2, 'nan', 368924000, 207), +('3fe82723-af0e-4df0-a3bf-32223b202dd5', 12, 50, 2, 'nan', 367552000, 108), +('6b22c272-f72c-4102-9abb-afff9d1c7d42', 14, 50, 2, 'nan', 371208000, 106), +('6738f416-1ce1-4cac-90ef-ead48b9b0ace', 150, 77, 2, 'nan', 375128000, 886), +('6ed39d69-1c01-42c1-8249-b8abf83c3608', 399, 844, 2, 'nan', 567664000, 508), +('d00fd8ee-7d65-469c-af2a-ca00681d224b', 1150, 54, 2, 'nan', 394760000, 160), +('42244303-b4a1-43d3-9041-f4f811691b41', 96, 81, 2, 'nan', 374352000, 118), +('4ac590b9-5733-4e20-b657-3b13d9b30a4f', 399, 843, 2, 'nan', 554660000, 491), +('1409a826-8176-49db-84f9-04d4cac0db89', 511, 146, 2, 'nan', 405100000, 228), +('21917565-5c24-4b9e-947e-0a7fc18b89dc', 1055, 102, 2, 'nan', 594684000, 424), +('cc4db5ef-c3d3-4067-98be-ee90a584a95a', 53, 71, 2, 'nan', 371640000, 1355), +('12f627b7-19aa-4583-bce6-a7f7d4dd66a2', 234, 45, 2, 'nan', 372792000, 119), +('0c6dc99d-d20b-4381-a405-8813129b81ee', 282, 54, 2, 'nan', 373592000, 255), +('449668bf-3294-42eb-9cef-d0c007b9b27f', 5924, 12, 2, 'nan', 400672000, 105), +('ee4c2079-e187-4a2f-ae9a-a95795a9d5fe', 476, 68, 2, 'nan', 446712000, 408), +('0f0d6f75-6ba3-4265-b53e-cc53a18e52b8', 3042, 13, 2, 'nan', 387236000, 117), +('d63bdad8-ace7-4d92-a8a7-add8620460d3', 353, 56, 2, 'nan', 376128000, 86), +('5a856042-6a60-4f9d-b256-d03295e2d7f9', 96, 82, 2, 'nan', 373728000, 100), +('dd9969e8-9efe-4781-af4a-6f2f8ff70808', 706, 56, 2, 'nan', 384544000, 202), +('37cff1b2-c11e-4511-8625-84def4635d84', 413, 42, 2, 'nan', 374920000, 112), +('36f312b9-5055-443a-b07a-2ab09e8f8cfa', 32, 14, 2, 'nan', 368144000, 119), +('de207a0d-932f-4a81-9412-5ff8baa9cdd5', 384, 87, 2, 'nan', 384584000, 122), +('b567274d-4631-4aee-85df-7d6603c20cfe', 287, 37, 2, 'nan', 391964000, 289), +('94d07841-1b75-42a1-9407-af6a341769b1', 5540, 32, 2, 'nan', 4107984000, 3302), +('d83a1dee-1451-4ea3-bd42-3881f1fa4e3e', 768, 87, 2, 'nan', 395492000, 126), +('3c736f54-7d2d-4cd6-9c45-8cce37ce5482', 112, 36, 2, 'nan', 372180000, 212), +('e600feed-52cc-4d8f-b28d-c6283dcde944', 46, 53, 2, 'nan', 368888000, 174), +('41624971-b40b-4e03-bb12-cce149cd6eac', 39, 73, 2, 'nan', 369812000, 108), +('9f769164-d25c-4e7d-b637-79f3489798c5', 30, 62, 2, 'nan', 368840000, 88), +('0f382217-32ed-4ea6-b7ab-c98c66e8305f', 291, 76, 2, 'nan', 378096000, 102), +('24d2cf39-25a1-486e-a763-f2639674f4b4', 116, 60, 2, 'nan', 367924000, 170), +('b303df36-67d8-4883-a109-294b61ccbe23', 113, 250, 2, 'nan', 381692000, 386), +('b46d8a4d-b969-4226-8719-6708c3755775', 391, 92, 2, 'nan', 379532000, 92), +('07b691e7-a940-4257-bea8-ebba6bc47637', 157, 67, 2, 'nan', 379060000, 126), +('9d624992-01fe-4ece-a7ba-ebf54da0eb7b', 33, 92, 2, 'nan', 369760000, 120), +('0aba4000-8b6d-47c1-a4fc-7b6a2b419b67', 384, 258, 2, 'nan', 421788000, 336), +('4635cab1-b1a7-4d6a-bb75-d95f0e1fd682', 24, 58, 2, 'nan', 333928000, 79), +('b2602d9e-7ce6-43e2-a4b3-3962f812a80a', 353, 149, 2, 'nan', 389096000, 219), +('f8d9c8b3-1e4f-4371-9d86-74778f7145a7', 150, 128, 2, 'nan', 376468000, 602), +('490bf28c-7f5b-428f-9127-c1ac19d611fc', 147, 23, 2, 'nan', 371728000, 92), +('a69254f1-d5d4-4f3f-991f-c197364b8187', 278, 35, 2, 'nan', 376404000, 116), +('702f4828-f604-44b8-a7b0-9684e77a0279', 64, 41, 2, 'nan', 366940000, 90), +('0d793328-c4c5-4739-b2f1-40afb4de55c8', 95, 37, 2, 'nan', 371568000, 249), +('7227a61e-59af-4b96-8919-efe13839889c', 4302, 62, 2, 'nan', 8104616000, 18008), +('57e351db-c504-42dd-a699-fe405c1b5a27', 1038, 37, 2, 'nan', 406872000, 245), +('88bbeb39-2fc5-4a46-8bd9-ca28a2fedef3', 399, 970, 2, 'nan', 612944000, 517), +('63a0101c-13b0-4c73-b4d7-00a33ae0f9ab', 119, 40, 2, 'nan', 320928000, 81), +('e3c17c93-5993-49fc-891a-9c9d5b9aa43c', 6, 11, 2, 'nan', 370792000, 194), +('6fd1fc5f-fce2-43c6-909d-8041a075ac3a', 537, 92, 2, 'nan', 418592000, 299), +('8ea8b437-819b-4624-b15c-a653b53c0a2e', 32, 41, 2, 'nan', 368692000, 89), +('c4ee461b-8d52-4338-8fa2-8a22b538456b', 12, 466, 2, 'nan', 374988000, 715), +('1eaa62de-c3ae-414d-9168-7fb5ece89949', 16, 41, 2, 'nan', 368992000, 239), +('6a63a491-c4c1-4176-9827-9460526ed319', 528, 76, 2, 'nan', 382696000, 99), +('733a21cf-e969-4a17-b888-820f2f9475a1', 307, 85, 2, 'nan', 384432000, 167), +('9a1e07e0-bac0-46e1-8b72-86a730f47c12', 862, 105, 2, 'nan', 456076000, 358), +('6ca8ca71-dbb7-4347-adb3-9ab02b69ef9f', 669, 95, 2, 'nan', 451828000, 523), +('07272b4e-88bf-4adc-9953-4b1f4cb3764f', 151, 15, 2, 'nan', 371468000, 83), +('8b9f9e1e-2e52-432b-8528-53f8b25ea81e', 2259, 34, 2, 'nan', 391664000, 104), +('6395bd2a-fd1b-448b-8771-f07a77958d84', 394, 48, 2, 'nan', 377592000, 265), +('b05bb0d8-a8bd-4634-82f9-47c52ccd83e3', 450, 140, 2, 'nan', 390568000, 125), +('38ee009a-7c6a-42a2-a05f-cd3f5286cd9d', 384, 92, 2, 'nan', 379920000, 107), +('d0799585-1e37-4134-b71f-481dc0d2be72', 384, 171, 2, 'nan', 402372000, 349), +('2d5a21fd-1cdb-4ee6-8e08-584bf99ef40a', 9, 12, 2, 'nan', 369064000, 152), +('a4aee4f4-91d5-46e1-9280-87b8ba7eeef9', 394, 50, 2, 'nan', 378484000, 120), +('7e361763-962b-4f8d-8c63-1a329428f89f', 384, 172, 2, 'nan', 406412000, 233), +('0abb0be4-b58e-4318-8624-4fc055f9d82d', 3938, 91, 2, 'nan', 532680000, 246), +('3497cd39-99dc-429e-8e10-67ecd7b2774a', 3, 79, 2, 'nan', 369232000, 193), +('0ca036c1-ceb5-4c33-993b-d8511a96208b', 78, 40, 2, 'nan', 369532000, 88), +('a22718ce-6e93-420b-89d6-cf2ddf104164', 57, 40, 2, 'nan', 369396000, 91), +('278179bb-f481-4d50-aced-5485993699a1', 95, 46, 2, 'nan', 369740000, 91), +('80272c6a-4312-4c93-9905-390e62a3d201', 384, 1747, 2, 'nan', 538556000, 2487), +('d5ab9552-64e1-43b2-bf46-5065612c2cda', 384, 1759, 2, 'nan', 1468608000, 2527), +('af467934-f4db-464e-8ef1-b6bbbbe88667', 394, 51, 2, 'nan', 376672000, 109), +('cfa62dd9-c6e0-48a2-9633-ee4d813d4f51', 148, 78, 2, 'nan', 371308000, 160), +('bd37012e-0b22-4e3a-b0ae-eba006d87749', 60, 34, 2, 'nan', 369272000, 119), +('d11bb684-448c-4916-a0f8-cd5748061638', 48, 15, 2, 'nan', 369052000, 163), +('46cb1988-f5b7-4a27-92be-dd8b7c7289db', 48, 15, 2, 'nan', 369052000, 147), +('dd000d70-ee4b-46a6-9e1f-26fd85b6df3c', 412, 11, 2, 'nan', 370240000, 122), +('a12d7860-89d0-4640-a4a6-73738b47408a', 11, 15, 2, 'nan', 368712000, 92), +('aae1e730-254d-401b-884d-d9f6965203f6', 11, 22, 2, 'nan', 369712000, 200), +('66e2dbf0-d223-48b6-bf1e-45a40f87d1b1', 45, 160, 2, 'nan', 363000000, 152), +('be2e6e7d-7992-4407-a7c1-aced14e6fa12', 112, 13, 2, 'nan', 370968000, 90), +('671071f2-380c-42a3-88f6-a1642c07dc36', 220, 68, 2, 'nan', 373128000, 233), +('5db2e298-251c-409f-81a2-844b9a2b7697', 1232, 61, 2, 'nan', 391532000, 206), +('9b0e1a4a-97a6-47fe-aa2a-6800d72fa611', 602, 72, 2, 'nan', 399216000, 309), +('9553c043-e844-4c9f-8367-eb5d7d51772c', 54, 64, 2, 'nan', 371940000, 141), +('78c3db71-ed10-457d-b048-d8142c535e14', 78, 28, 2, 'nan', 369532000, 96), +('60a72e37-28f8-4288-81eb-830cb2924a03', 76, 33, 2, 'nan', 370068000, 198), +('520e37c3-6e64-4fce-8a88-cceac2b3b5b8', 80, 68, 2, 'nan', 372164000, 139), +('596f24ae-49e1-4a22-bbee-0b53b1bc4b26', 921, 486, 2, 'nan', 1131140000, 2776), +('7fce7f74-0c73-4fc2-b5bc-45f570014054', 192, 32, 2, 'nan', 400300000, 270), +('0061bb8f-219c-4e53-8ea5-137799e5fc3c', 95, 32, 2, 'nan', 398676000, 134), +('09c27409-4fb5-415c-93e3-7e2233e0109e', 115, 61, 2, 'nan', 398460000, 111), +('1a95492b-e852-413b-8450-72eb0adcd577', 460, 61, 2, 'nan', 401832000, 98), +('c6315300-f68e-4bf7-b7d2-778c3409f346', 939, 174, 2, 'nan', 717876000, 917), +('517dd44b-61bc-47ba-a7c8-c189378f6d73', 2591, 206, 2, 'nan', 1137112000, 2656), +('c167f049-4d43-4409-84be-233b44981bb8', 286, 81, 2, 'nan', 400352000, 111), +('2c4290f6-e59f-438f-9664-a22810b495a0', 22, 67, 2, 'nan', 395520000, 114), +('e7a1816e-c98c-4ae9-854c-b034d4dc77cf', 16, 77, 2, 'nan', 394388000, 99), +('e5a06530-b313-4b75-ad14-5bceeb690d5a', 6, 67, 2, 'nan', 394636000, 110), +('8d671718-8686-4cc5-899f-178a2fba763e', 168, 59, 2, 'nan', 399172000, 184), +('ef88e6d6-affc-4511-8842-af8f32c1b49e', 40, 56, 2, 'nan', 395716000, 109), +('a1e85eec-0ed6-43ad-b0b0-d50fa4a26ed8', 24, 56, 2, 'nan', 393124000, 115), +('79baff1d-1f54-429f-91b2-4bc4e6ed5c18', 294, 64, 2, 'nan', 401648000, 253), +('a318ddfc-153d-4462-b7f9-963f7b3a11b2', 8, 41, 2, 'nan', 394628000, 257), +('8d376621-fad2-4329-9a54-5cc8a5347e9b', 287, 19, 2, 'nan', 416688000, 208), +('0a78d681-284e-4898-a80c-271edad7b1d3', 44, 43, 2, 'nan', 395092000, 153), +('0cf2846c-a2cc-4b4c-bc53-b78bab20d678', 168, 65, 2, 'nan', 397236000, 305), +('43ad2854-6f47-43ad-8b8f-4e77591598c0', 143, 33, 2, 'nan', 397416000, 439), +('fc591b9e-f6df-457c-ac61-d3ddc8a97df6', 30, 43, 2, 'nan', 397724000, 105), +('3b0ee8e6-d921-4c11-a19a-e482abff09da', 168, 67, 2, 'nan', 398316000, 141), +('efda7c49-2865-4bf9-847a-46ec0ed48c9e', 192, 486, 2, 'nan', 471312000, 466), +('1ef05ca4-74d6-4cb1-a0dd-64412a5e8f26', 88, 1758, 2, 'nan', 710156000, 216), +('f8a29cc6-a0cb-458e-938e-b30635035f74', 192, 485, 2, 'nan', 473932000, 486), +('69187ffb-8155-4c06-baa9-69b3c53f901e', 60, 57, 2, 'nan', 396240000, 519), +('d3d1f2f0-6462-453c-b346-21ae50813c3b', 192, 125, 2, 'nan', 407884000, 128), +('f0b99e31-aa9d-4111-b2a5-0ddcb7a2c063', 192, 67, 2, 'nan', 400544000, 166), +('449d5f69-50db-4c11-850e-ae578bdc68ad', 288, 97, 2, 'nan', 404500000, 286), +('108c14b7-9be9-4107-be16-4e70ae0bdb8d', 778, 68, 2, 'nan', 586496000, 225), +('3161c5b5-05c9-4513-bbc1-9b7cfbac6275', 96, 92, 2, 'nan', 398024000, 109), +('69fcd88b-6edf-4cde-a610-0363effd0097', 158, 66, 2, 'nan', 398412000, 113), +('a9b8ad78-8f0d-4352-aa0d-c9fb4eab8c2a', 60, 56, 2, 'nan', 395452000, 393), +('619fa191-ec16-46f2-86bd-6d8ef583db5f', 168, 66, 2, 'nan', 398248000, 93), +('6779287f-41eb-4733-977e-8bd8e3991a24', 24, 10, 2, 'nan', 393820000, 174), +('ba770cd8-8667-40c9-b97e-c714cda2f80c', 30, 40, 2, 'nan', 395556000, 129), +('58f871c7-830c-4d71-9300-e3779e203c5d', 80, 41, 2, 'nan', 396468000, 104), +('15b23bef-7bdb-4361-891e-c8a259f22230', 80, 34, 2, 'nan', 397080000, 216), +('246ebe66-68e4-4a9f-9a97-06f881c83cbd', 19, 17, 2, 'nan', 395288000, 105), +('99529852-6a5c-4d5b-b029-3455aa3a8c8d', 862, 79, 2, 'nan', 425444000, 261), +('1a8cffdc-13d8-4779-a38f-a9abeb854e44', 173, 80, 2, 'nan', 399860000, 894), +('144380ec-debc-468a-9f7a-930197d9547b', 233, 50, 2, 'nan', 397660000, 306), +('6365416f-e395-45d2-b558-270d0992fa0f', 18, 10, 2, 'nan', 392916000, 143), +('eeb06d17-12d1-45c1-ac8d-4e7473983ebf', 80, 60, 2, 'nan', 395456000, 183), +('4c38dfa2-b27b-4409-ba90-2c75eea4bba4', 233, 53, 2, 'nan', 397488000, 93), +('a70578ed-87e9-4ef7-a1d2-760d76962e40', 216, 58, 2, 'nan', 398032000, 193), +('eaffc15a-0d71-427b-b672-0487943d7da7', 16, 57, 2, 'nan', 394104000, 131), +('c86955d9-980c-46be-af88-26d4ac2d2a7d', 25, 11, 2, 'nan', 394264000, 313), +('7f7fd4fe-d7df-4e7d-a37c-936bc8377f5d', 20, 41, 2, 'nan', 394172000, 142), +('ba61cf3b-6c70-456d-8e9e-c8541a2a6407', 579, 42, 2, 'nan', 473148000, 321), +('ed986358-8a50-4b19-af8b-d5df11595183', 272, 38, 2, 'nan', 397624000, 165), +('70f9bec4-9459-46f5-aeab-ac3ac7f55cf1', 10, 36, 2, 'nan', 390764000, 78), +('38c42216-080e-413c-8c79-f3a27ab556bc', 32, 19, 2, 'nan', 391968000, 82), +('d3c8f668-6393-4e41-8160-a17e516ff500', 24, 466, 2, 'nan', 400796000, 222), +('692c4d20-72b3-4627-8dad-7379174604a1', 304, 26, 2, 'nan', 397456000, 209), +('3382c6a9-b778-4a75-9092-6c76afeeb45e', 16, 56, 2, 'nan', 390720000, 105), +('bbfa6b71-fd64-45a3-a66e-5d0d7f454e9d', 96, 125, 2, 'nan', 400764000, 170), +('27b552f9-b3be-4fbf-93dc-323774590e1b', 768, 94, 2, 'nan', 534576000, 860), +('4af523ba-0276-48a3-bfac-246ff361310b', 384, 105, 2, 'nan', 413812000, 160), +('66630aa4-c73b-4dbd-9290-b5a6ff8d0129', 137, 121, 2, 'nan', 399664000, 149), +('9e5aa6a4-23f4-4949-82df-748e4a651d62', 137, 121, 2, 'nan', 399664000, 162), +('b6965426-898a-46e1-a40f-2fe34a9d7bb1', 24, 41, 2, 'nan', 394588000, 95), +('052163a0-2807-40fc-8ae7-ecb8c4c1d421', 296, 108, 2, 'nan', 405108000, 118), +('bd0b608a-05cd-4c67-a4d9-e3337075e2e4', 36, 19, 2, 'nan', 394524000, 107), +('a5d9bb39-116b-49e3-97c4-831580f6729f', 399, 50, 2, 'nan', 406772000, 168), +('cefcf3fb-26f2-4349-9fc8-d81c63e928ff', 146, 60, 2, 'nan', 397560000, 115), +('c573d2c7-91bd-4dfe-a433-a6ef89a9656c', 1093, 114, 2, 'nan', 526824000, 646), +('869f9610-e808-4935-a57e-781abc9960af', 286, 14, 2, 'nan', 329000000, 201), +('83c880c1-20e6-4134-aca4-ced663066543', 96, 146, 2, 'nan', 332600000, 178), +('cd376889-4d87-40f8-8d3e-b9a4c0623f55', 183, 20, 2, 'nan', 301320000, 76), +('d8d739f2-6d56-41b2-a5a7-38133f9d7bbd', 192, 34, 2, 'nan', 328240000, 251), +('be224c35-461b-48a3-ab48-4e4838019c64', 10, 53, 2, 'nan', 323908000, 171), +('8b99435f-643a-4347-b552-18a5aea558e0', 576, 104, 2, 'nan', 372232000, 257), +('9d372971-7763-4731-a44e-cb87a3434cc0', 355, 42, 2, 'nan', 333436000, 125), +('9a3a5d72-c2b0-449a-a157-1c088e0af56d', 511, 140, 2, 'nan', 408944000, 348), +('a8afae6b-c328-48df-bce0-bab522a98f85', 168, 31, 2, 'nan', 327776000, 130), +('1fa748d3-fa80-4af4-a8fa-7782a27fa585', 1728, 121, 2, 'nan', 446964000, 371), +('559da3f1-bc29-4c26-a6ac-06b52b054594', 985, 27, 2, 'nan', 349308000, 149), +('cb41a3f9-0be7-4dad-b015-68016ef51022', 94, 34, 2, 'nan', 326732000, 155), +('19203be8-1052-4c80-9bce-fff4e77f05c8', 192, 86, 2, 'nan', 332644000, 88), +('a91d644d-9c86-411c-9ae2-4eeac7545d2b', 96, 78, 2, 'nan', 329440000, 118), +('5e0bba68-4bfb-40e1-96d1-daf1a454fd46', 355, 41, 2, 'nan', 333388000, 81), +('510e59c2-ba8e-4b1c-b4be-17dc59c80457', 192, 103, 2, 'nan', 333004000, 94), +('a4b6745c-bad5-470a-8e32-2f75236c2fce', 19, 80, 2, 'nan', 326460000, 180), +('f69d7684-607e-4661-a836-c09aba157c95', 0, 0, 2, 'nan', 101468500000, 275814), +('5c679aad-68f7-4855-be14-8312110bc173', 273, 46, 2, 'nan', 332156000, 151), +('e015a0a1-5c4f-4178-8ebc-2b9ac9e4e8b6', 108, 22, 2, 'nan', 326384000, 94), +('cd2374bc-649c-45e2-8540-991fb36cb487', 198, 72, 2, 'nan', 329712000, 95), +('7c707855-1640-42e9-82a7-d720e58874eb', 4952, 780, 2, 'nan', 8902036000, 17603), +('20af2ade-f786-4305-85b1-b5272ab356e0', 4566, 177, 2, 'nan', 9945200000, 68222), +('490b3500-dc03-4fa9-9cd2-78126b9333c2', 372, 131, 2, 'nan', 336100000, 75), +('e5504565-cf0e-4f2d-b925-1c2ca7cb5fbb', 353, 143, 2, 'nan', 343548000, 145), +('f9a9e2d7-b568-45d6-89b8-89cfe9991f77', 96, 27, 2, 'nan', 326760000, 83), +('8500b61a-9f08-49c9-a897-2773f4f383c5', 12366, 782, 2, 'nan', 54234712000, 114505), +('7e8480aa-af3b-4df7-8d66-8f93a28ccb74', 27079, 17, 2, 'nan', 1158688000, 635), +('f87a89b9-4b6e-4398-8e48-0b6ec238cc97', 4, 12, 2, 'nan', 325740000, 862), +('92b6145c-1248-4024-b825-a8b6cb07c4b0', 4, 13, 2, 'nan', 323444000, 228), +('04e67d15-1907-4fd1-9a4d-48b47f6c71f8', 18, 71, 2, 'nan', 326436000, 128), +('f7466925-9a5c-40d8-9536-9c43ad30cc2d', 80, 63, 2, 'nan', 327592000, 96), +('2f21b2ec-91b2-43e2-8d70-b35b64fcaf0c', 768, 224, 2, 'nan', 543720000, 2123), +('3df051d5-99a8-411a-b40f-bc269eeb5a6c', 1536, 224, 2, 'nan', 587236000, 1428), +('6048e851-42a4-4b0b-b040-a05cfe9686b0', 560, 48, 2, 'nan', 339844000, 120), +('06bbba17-3761-4e80-a7c0-56a01e92e7f2', 8, 20, 2, 'nan', 326012000, 82), +('1d1f338a-0b0b-4597-a960-1ecf5019dcfe', 92, 63, 2, 'nan', 327720000, 85), +('ba1c6fdd-bedd-45f1-a255-aa6b5e21fbd7', 44, 45, 2, 'nan', 326472000, 83), +('1f0a1ce3-35b9-427a-b11d-096690fe4a27', 30, 42, 2, 'nan', 326428000, 268), +('194ec434-bedd-4403-9e14-2351ea3c7720', 1392, 189, 2, 'nan', 576900000, 825), +('633b6978-0c6b-400a-b466-f3eb7e2035e9', 80, 65, 2, 'nan', 326916000, 129), +('dd18384b-c038-48bf-b751-573163a4223d', 2591, 200, 2, 'nan', 1092796000, 3246), +('635c4f2a-8417-4fae-9078-e94846c8b7b1', 3822, 78, 2, 'nan', 5082892000, 6137), +('759be360-787b-410e-9e07-7db03e4df5c6', 14, 11, 2, 'nan', 325624000, 83), +('37d514e8-ce44-4163-b47b-f0af05397aff', 264, 70, 2, 'nan', 332012000, 95), +('7b0f4fc2-d772-4921-ad44-ba64f148a84d', 792, 70, 2, 'nan', 341216000, 100), +('9a730dc2-23b7-4c41-8270-5fa7af22d468', 5, 51, 2, 'nan', 326028000, 126), +('c033bd49-d4e9-4df3-9317-ce030d4b80d6', 320, 41, 2, 'nan', 331744000, 83), +('09ee747d-638c-4933-bbe6-4e999d4b7480', 960, 155, 2, 'nan', 606116000, 489), +('16060ef4-c199-493f-ab88-d4b4f06138a4', 155, 20, 2, 'nan', 330572000, 133), +('8c1f6f46-75e6-4439-a5c0-c4e9bad71ec0', 113, 36, 2, 'nan', 331016000, 116), +('22c6824e-2223-4a56-8686-dcbf3db8b2f8', 306, 75, 2, 'nan', 333324000, 83), +('45c822ad-7817-48b6-8208-7b65922a5b01', 263, 72, 2, 'nan', 331556000, 96), +('40e38e61-5426-4495-95a8-9bfd717b779e', 89, 68, 2, 'nan', 327840000, 122), +('6b7f13b9-d019-4cd7-9913-6cc2a2d558c0', 155, 37, 2, 'nan', 331360000, 142), +('643a6bdc-48a3-4624-ab93-1e70053c0f06', 155, 38, 2, 'nan', 332204000, 250), +('d97b7b26-6c93-4d55-bd52-c4beae6fd9fc', 14, 46, 2, 'nan', 326240000, 84), +('caab8faa-5ebd-4734-91d3-cc598fe332c4', 38, 31, 2, 'nan', 325800000, 169), +('a005ae56-c6a3-4e0f-8280-49d85e32023e', 384, 85, 2, 'nan', 356464000, 166), +('1a93bbf7-b118-4b67-9477-e5a067291f7e', 1359, 85, 2, 'nan', 1048300000, 1971), +('670fcf04-077d-41cd-b9b6-d5eb7b37b42a', 32, 38, 2, 'nan', 326332000, 85), +('97704373-2097-4e0f-aa15-5a6087877b86', 1216, 88, 2, 'nan', 612756000, 600), +('3f1f3955-04a0-4554-ad87-55b4c2b13ee4', 27, 38, 2, 'nan', 300956000, 80), +('97d9b7c0-b68d-41ff-954c-3755e6ff1cb9', 14, 31, 2, 'nan', 325888000, 161), +('15f391a5-8c92-46e0-a321-5b981848d586', 231, 101, 2, 'nan', 341364000, 154), +('e0d0e1ad-55f7-4486-a53a-7df8b44bd4ab', 320, 44, 2, 'nan', 352328000, 207), +('b5415610-17a1-4ac7-a985-a060df556451', 68, 43, 2, 'nan', 329320000, 97), +('80dc4574-8278-4492-9473-f6091c52b98c', 158, 85, 2, 'nan', 332248000, 527), +('f838c5ba-bc20-4c5c-94cb-52973b522be4', 122, 264, 2, 'nan', 337212000, 284), +('5a1d5e55-4fe4-4dba-9406-684ffb734c5f', 382, 94, 2, 'nan', 353788000, 178), +('a5f23b08-8499-4e16-9b6c-69d089c9fd09', 307, 80, 2, 'nan', 350960000, 186), +('2b365ffc-a96b-47a6-89c2-6d2aaae535df', 31, 80, 2, 'nan', 326692000, 121), +('32a9bb7f-56c2-4bbc-9120-d66d4d8debbc', 862, 94, 2, 'nan', 422736000, 397), +('2a1d211e-5b53-4121-93f6-49721523eda4', 2203, 76, 2, 'nan', 2450852000, 6303), +('bffc8bce-1f6f-4bff-ac7f-8dad7f43895a', 231, 107, 2, 'nan', 339048000, 141), +('fc86707d-54ca-4eee-92d3-dbcde06ed75c', 123, 50, 2, 'nan', 327536000, 81), +('0c7ddae6-83d6-4db6-9a79-9720c614142b', 424, 201, 2, 'nan', 425668000, 587), +('9194b7f9-a420-426e-8073-78ad6607cd1d', 39, 81, 2, 'nan', 301268000, 150), +('db9b1f49-f848-4a40-bbbb-ed9d3848965a', 39, 81, 2, 'nan', 301236000, 150), +('84eff552-5723-4729-a86b-8f283874c289', 62, 83, 2, 'nan', 328500000, 169), +('b251dfae-0bb8-4bee-ab47-bdcc17d25b62', 16, 83, 2, 'nan', 325920000, 211), +('316b9754-2d5d-4822-bd72-f86f6ca9ae4b', 96, 152, 2, 'nan', 331648000, 186), +('d509e593-4053-406a-8ab0-8af8b1b4c8a5', 231, 97, 2, 'nan', 372816000, 612), +('e66faf00-0ffa-404f-ac08-7121c6337b70', 242, 43, 2, 'nan', 368180000, 297), +('66782585-883b-4784-a443-a2dfff848d88', 152, 83, 2, 'nan', 370720000, 348), +('9291cc5a-9ab4-49c0-b8a1-9fc9b6c46b44', 92, 83, 2, 'nan', 371068000, 420), +('decafc1b-5c70-466b-a8cc-33f29c6db3df', 12, 83, 2, 'nan', 366976000, 127), +('5b319607-4bec-4e36-ae54-5b7c21e91064', 336, 22, 2, 'nan', 366464000, 97), +('2f3f309f-51bd-43b8-80d1-10d10a1c274e', 152, 66, 2, 'nan', 371584000, 210), +('893621a8-d4dd-464b-8977-223470fc297e', 152, 81, 2, 'nan', 365052000, 234), +('12e91952-b2b1-424d-884c-865e77474bf2', 28, 52, 2, 'nan', 367340000, 316), +('d6946499-e2e7-4286-a95b-786b8ec633a9', 44, 83, 2, 'nan', 367956000, 265), +('1fa39acc-83ef-409f-91d5-13ef1934635d', 32, 64, 2, 'nan', 369696000, 240), +('694a13a5-9e5d-4f54-b34c-a63167a045b1', 56, 80, 2, 'nan', 370264000, 251), +('c4ff14f9-3a89-40d6-aa85-1fbd8dd80c17', 138, 89, 2, 'nan', 371388000, 347), +('d12d6d89-a8e9-4fd9-8e1c-16968035fb38', 152, 72, 2, 'nan', 372660000, 201), +('9627e0a4-8f47-430e-b1d2-30c1295a722f', 32, 83, 2, 'nan', 367744000, 237), +('f50accb5-3461-4a7f-9bff-761e4414a026', 28, 51, 2, 'nan', 367140000, 156), +('e0d4954f-e964-4755-bac3-bc47610206aa', 21302, 90, 2, 'nan', 30206900000, 193946), +('2d60289d-1b7f-4945-8264-b01e2426c23e', 4566, 178, 2, 'nan', 5398704000, 12040), +('179d0c15-5af2-4671-b807-fb374fd3b842', 307, 85, 2, 'nan', 373952000, 195), +('f6a8189a-caa7-4a41-bbc0-09ac46bf5a8f', 1067, 12, 2, 'nan', 370924000, 112), +('59ca6817-c388-47ad-ba64-47989313dd44', 6374, 49, 2, 'nan', 9769896000, 22148), +('61d6d576-362c-4261-859e-7209fa346f1e', 152, 85, 2, 'nan', 370008000, 194), +('bd89bfe7-51b0-429b-9a88-2e71c8cb7545', 856, 49, 2, 'nan', 481572000, 597), +('c5cc0f29-0fa0-4c44-b33c-103f0e3b6f32', 39, 93, 2, 'nan', 371980000, 199), +('53db452b-9234-4830-94ef-cf9ab49ef0d1', 150, 63, 2, 'nan', 372480000, 494), +('a268c541-fe8d-4521-8a49-e3a8439aef64', 394, 42, 2, 'nan', 378392000, 1151), +('387f71b9-3283-4923-860b-8065a62248fd', 189, 142, 2, 'nan', 380964000, 373), +('8194c64a-1a43-499f-8f47-3a3935c557f4', 856, 48, 2, 'nan', 521784000, 784), +('294d4a3f-f8ff-4644-ab15-3ddcb4390af1', 177, 384, 2, 'nan', 379924000, 284), +('a373adcd-9b62-41e3-9ae4-a6f79ff45f45', 96, 99, 2, 'nan', 372380000, 265), +('ef7406a1-a4fa-4af2-8e67-92607ebdd8f9', 1005, 137, 2, 'nan', 410232000, 168), +('59dbbe08-8701-4e48-95ab-3882a02ee1e7', 413, 40, 2, 'nan', 405220000, 243), +('088a65e5-c848-4e34-b47d-4664f9f27f83', 150, 77, 2, 'nan', 374532000, 1131), +('45d538e2-97c5-4893-af08-f6bf04abb767', 1150, 54, 2, 'nan', 390968000, 368), +('0b9706a2-e0a6-438f-a5b3-a094161a15a7', 399, 844, 2, 'nan', 539092000, 751), +('eb3650f0-7aea-4cab-a520-6e581e3bd978', 399, 843, 2, 'nan', 539232000, 1134), +('c6635e7e-93c0-403e-b615-3e8f7da29352', 1055, 102, 2, 'nan', 554628000, 434), +('2cb50802-1ac7-48a4-bebd-4d9be546cf69', 413, 42, 2, 'nan', 374312000, 681), +('1c6d9feb-17fb-40ca-a3f4-d1e7834c5d53', 96, 82, 2, 'nan', 370640000, 197), +('ed2a60cf-2164-4b01-a59c-07518613ef8d', 282, 54, 2, 'nan', 373164000, 346), +('469e8fcc-37ef-4dfb-b4cd-2b109200ce4c', 5924, 12, 2, 'nan', 394020000, 112), +('52802b61-7df2-4e82-9ccf-5c8075c2da45', 96, 81, 2, 'nan', 371540000, 130), +('0c715e72-47dd-48ac-b733-a845e440e86a', 476, 68, 2, 'nan', 426996000, 469), +('13f6b03d-40e1-47c2-a7df-163c0e3e7d23', 353, 56, 2, 'nan', 225308000, 475), +('9639186c-9084-47ed-8075-3f14798eceef', 384, 87, 2, 'nan', 384048000, 1021), +('6e54aa84-ffd1-4943-8ae2-7e75319a7e30', 287, 37, 2, 'nan', 371328000, 585), +('dba3182c-80c0-46ed-a9e6-64e0cf699741', 39, 73, 2, 'nan', 369720000, 195), +('b33244f5-30ce-4db3-94c7-4dcbe5cff4a7', 353, 149, 2, 'nan', 388952000, 263), +('ba196947-c2d6-4ae1-ad23-fb768c087125', 511, 146, 2, 'nan', 388392000, 235), +('08d92acd-992c-4500-a33a-094225dec9cb', 147, 23, 2, 'nan', 371064000, 93), +('b0bfe252-9f89-4160-839e-738a649de0f1', 95, 37, 2, 'nan', 370196000, 252), +('ea9e5e73-c141-4cf6-944a-6f43541ed747', 768, 87, 2, 'nan', 392580000, 269), +('b6d61911-54b5-4917-957a-11f5931f1b6e', 399, 970, 2, 'nan', 600196000, 527), +('3cf83989-1316-47f4-a291-03aa41da31f1', 32, 41, 2, 'nan', 367032000, 92), +('531b6413-c293-4a87-9444-3939eebf79bd', 528, 76, 2, 'nan', 382568000, 104), +('18919546-fade-4e83-b5dc-1708043bec82', 151, 15, 2, 'nan', 367920000, 90), +('6e6eb332-9166-4059-8a7c-cc4dfa5417f3', 394, 48, 2, 'nan', 377028000, 283), +('c36b0187-97f8-49c4-9f78-dfb75a04f41d', 450, 140, 2, 'nan', 389624000, 126), +('f9552ba6-db05-4a74-acb7-f851c627777d', 384, 92, 2, 'nan', 379340000, 200), +('4777ec99-b163-466f-81a8-a5e5f3c81b5d', 862, 79, 2, 'nan', 398504000, 301), +('3429c0f3-9f05-4234-a3ec-ead9285d9c91', 384, 171, 2, 'nan', 400700000, 448), +('a9e4fa1b-5f48-4143-a06e-20c6013c37f0', 9, 12, 2, 'nan', 368400000, 193), +('eb734567-fa4f-4aff-8ac4-e119b7cf31cc', 384, 172, 2, 'nan', 403540000, 337), +('52b3c3ce-0681-4992-9cd5-bdb0eddffff6', 3938, 91, 2, 'nan', 486604000, 379), +('ddb6e8c1-700b-4c2e-834b-26ceca5be035', 3, 79, 2, 'nan', 368916000, 219), +('ed55d400-04c2-4aa7-be76-63eef565cbae', 3, 80, 2, 'nan', 368524000, 193), +('23d23e6a-7e1e-41e6-aedb-5c997c6a31a6', 384, 1759, 2, 'nan', 1362084000, 2677), +('160160e6-ee83-47f7-8d59-1e6df6dd1a4d', 394, 50, 2, 'nan', 376560000, 377), +('e3b07a6e-145b-4551-b27c-905a1601a1f3', 60, 34, 2, 'nan', 368544000, 198), +('462b169a-acce-47a7-adf9-1daa0a465639', 233, 50, 2, 'nan', 373160000, 382), +('a43beee6-aab4-4ddd-895e-b7f95e258d17', 54, 64, 2, 'nan', 370380000, 308), +('4df02286-2ecd-4dd0-9d13-ec5d6b2e76d9', 11, 15, 2, 'nan', 344776000, 108), +('50b68208-0bdc-4ecd-9f75-2036a2332d55', 11, 22, 2, 'nan', 369672000, 201), +('9bba65d6-12be-4f4b-b40a-018489ebbc3b', 35, 42, 2, 'nan', 368940000, 256), +('40fe74d9-7fa2-4ef1-a64d-c092b3e5128b', 1232, 61, 2, 'nan', 391368000, 967), +('716b59bd-d2f0-404e-b6c5-ad5b69e8db3a', 78, 28, 2, 'nan', 365844000, 152), +('cd5299dd-f160-4b8e-9eac-ea2cb54c003a', 78, 28, 2, 'nan', 362160000, 152), +('6140d69a-3c93-4378-bc99-fc0596aa6703', 76, 33, 2, 'nan', 369936000, 299), +('c03cd275-e2c0-4e28-a0a1-e07be2c24c03', 602, 72, 2, 'nan', 398576000, 1664), +('9afa1b4a-244e-44a4-8b9c-69c229007667', 80, 68, 2, 'nan', 370328000, 565), +('655e4674-a14d-485b-9294-18d8e76014ca', 921, 486, 2, 'nan', 636868000, 2955), +('2a009904-eb9e-49bd-bee0-da0bb1bfdba5', 192, 32, 2, 'nan', 399972000, 280), +('186f76c0-c877-428c-ab08-46cbcf4524a1', 115, 61, 2, 'nan', 387000000, 210), +('f5d2cdc0-0061-4a7b-9c86-bee6d7e08d29', 460, 61, 2, 'nan', 400424000, 114), +('f147fe13-56a8-41db-b399-ecabba242ca9', 2591, 206, 2, 'nan', 896908000, 2792), +('0bef5c46-6ad7-4a05-8c4b-8d227f0a0352', 22, 67, 2, 'nan', 392356000, 193), +('fc0ebaac-1bbb-4fb6-b20d-2fe74f31b5ef', 16, 77, 2, 'nan', 393268000, 112), +('fcd7e83f-804c-40bb-9f3d-8483cffe34c0', 95, 32, 2, 'nan', 395920000, 497), +('284cc76f-487e-4d49-9c2f-d4d10e74385e', 168, 59, 2, 'nan', 398028000, 416), +('b8e3c3ac-5b5b-4f6d-97c2-3296b0bab7e6', 40, 56, 2, 'nan', 395220000, 114), +('9262c5ec-b92b-425b-8fd1-10ea4905d595', 24, 56, 2, 'nan', 392936000, 117), +('7bae4416-a699-4985-9736-60bf2b4aa4d6', 8, 41, 2, 'nan', 394360000, 276), +('7cb7e7e1-48c1-4259-9856-395911e88d4d', 143, 33, 2, 'nan', 395756000, 679), +('b127f3dc-fc5b-4054-a20b-e9216caff4ef', 192, 485, 2, 'nan', 455936000, 698), +('dfb12c24-d67a-4dc4-b221-b6fb1e49602b', 287, 19, 2, 'nan', 393648000, 28278), +('507da6a3-0df4-4e30-9763-f392eb62aac5', 168, 67, 2, 'nan', 398216000, 371), +('6fe5c170-4774-4576-813f-7c164866a3bf', 88, 1758, 2, 'nan', 709548000, 216), +('3c31aad8-1e86-48be-b65c-4924388bd587', 192, 486, 2, 'nan', 418000000, 1123), +('2b2e2473-fbc6-4e6a-9072-19f6411b0817', 192, 67, 2, 'nan', 400156000, 170), +('5994ad72-2f1a-416b-ba3c-9cc4a1c61c20', 778, 68, 2, 'nan', 539272000, 227), +('ba111653-1c15-4425-8ab9-11a5422eff1d', 96, 92, 2, 'nan', 397212000, 188), +('fdec2e67-e193-4185-991a-be136e63b2b0', 158, 66, 2, 'nan', 396860000, 318), +('6218ce6d-5208-4ebc-9e48-633b3f54cdcc', 60, 57, 2, 'nan', 388668000, 26827), +('d6eef75a-4505-4111-9c4d-5a748d403dd2', 60, 56, 2, 'nan', 395448000, 561), +('7a24dba0-f6b4-48cc-b1cb-4c9b884170e6', 168, 66, 2, 'nan', 398184000, 153), +('ff077c77-a1ba-46ac-b5ca-d5c894242770', 80, 34, 2, 'nan', 395212000, 228), +('65526acd-86e9-421b-9c91-b885cd36729a', 19, 17, 2, 'nan', 395088000, 166), +('04d09a9f-2151-47df-8475-36fc8ec7aafc', 146, 60, 2, 'nan', 396424000, 784), +('7e72e913-ce87-4bc1-adf7-c3cc7d5184e5', 233, 53, 2, 'nan', 397100000, 395), +('9e6f2ff5-f749-4089-b828-ef29fcca1a7d', 216, 58, 2, 'nan', 396236000, 227), +('335635a7-8153-4e4c-9b53-e1bc145497b0', 16, 57, 2, 'nan', 373140000, 156), +('5452c17e-ae82-4b46-a85a-ece0501a5d5e', 20, 41, 2, 'nan', 393884000, 214), +('f5a06f01-7405-4ff1-b047-64f9e7505d6c', 579, 42, 2, 'nan', 442784000, 383), +('bf9fc8f3-39b7-4771-95e8-f858494022a5', 272, 38, 2, 'nan', 397296000, 30202), +('98270055-d47a-4cc4-b40c-ac006f7823f6', 24, 466, 2, 'nan', 400612000, 234), +('009b95ec-18b6-4fa0-a0bb-0785cfe03409', 304, 26, 2, 'nan', 396920000, 211), +('89ae9ba5-f920-46e6-9279-40d5be29f845', 36, 19, 2, 'nan', 394500000, 198), +('0f24f638-13fe-4898-b3a9-b15aafb260f8', 399, 50, 2, 'nan', 404516000, 174), +('37d12cdd-c03b-4921-a77b-851269d3e557', 1093, 114, 2, 'nan', 482504000, 654), +('9308c777-1c30-43ac-acf6-9454601601ba', 96, 146, 2, 'nan', 332352000, 245), +('a1910b9b-2d93-4239-9bc1-1250fea56837', 96, 146, 2, 'nan', 331548000, 245), +('1280dc5a-d797-42ca-a613-3fbce32f55a0', 14, 52, 2, 'nan', 300932000, 166), +('0a329ce1-1b82-4faf-a1f7-0d17e4b25251', 576, 104, 2, 'nan', 371176000, 260), +('3b212797-bb0b-422c-bba1-79921ce40c78', 168, 31, 2, 'nan', 327104000, 142), +('c14e97a2-a083-4e49-a14b-0abd23483272', 1728, 121, 2, 'nan', 446280000, 376), +('7cd84e00-773a-4387-bcdb-37da819ce5f7', 108, 22, 2, 'nan', 315832000, 140), +('8553c8e2-5bc9-41b6-9f5e-96a2abfb7055', 192, 34, 2, 'nan', 302704000, 255), +('65212822-529e-4cf4-a427-78032b9e569b', 198, 72, 2, 'nan', 329532000, 209), +('3a8d8b29-d408-45ab-8d63-043741de9679', 273, 46, 2, 'nan', 331556000, 181), +('d68f1265-c669-4f2b-a3ab-ee0e547e0143', 4566, 177, 2, 'nan', 9765660000, 70248), +('f0b0c5aa-9f97-42e3-82e4-71968f28ad15', 511, 140, 2, 'nan', 401220000, 585), +('1bf64e7e-4ab9-4dfa-9aeb-6402ad95c165', 353, 143, 2, 'nan', 343172000, 439), +('605f6954-8697-42c4-97a4-7c3aa6b3cde9', 96, 27, 2, 'nan', 326164000, 182), +('3688f502-6895-4041-b957-6487f6ba7c98', 80, 63, 2, 'nan', 325924000, 123), +('013f8041-6f36-4dcb-8af9-67ac1fd78d9e', 8, 20, 2, 'nan', 312564000, 94), +('58e69a18-ecef-4b61-a62f-7efc1f733731', 8, 20, 2, 'nan', 312896000, 94), +('e7cb303d-ffe0-4a5e-8eb0-627dd0513f51', 1536, 224, 2, 'nan', 562312000, 2121), +('7cc4a810-137c-468e-a249-60b0faf9a003', 560, 48, 2, 'nan', 339180000, 120), +('056330e4-36eb-4c38-b87f-56484ac69623', 92, 63, 2, 'nan', 326952000, 88), +('ea9e5c11-f7a7-4dcb-b8b5-20b499611b9f', 30, 43, 2, 'nan', 326568000, 2029), +('8df2b291-adbf-4def-a172-ece656410050', 1392, 189, 2, 'nan', 542424000, 825), +('ce47b98d-8a78-4e97-b20c-297f75394f28', 80, 65, 2, 'nan', 326612000, 184), +('d8e90f5f-1b10-4f00-9c3b-6bc8daf2cc8a', 44, 45, 2, 'nan', 325972000, 95), +('cec46141-0286-4430-ac41-ed830decb5e8', 12366, 782, 2, 'nan', 54233492000, 164399), +('fc9b54c3-0447-49db-a6d0-5b913d0e285a', 264, 70, 2, 'nan', 331724000, 147), +('3e6d362e-3880-4b90-b693-868547ce0f5a', 792, 70, 2, 'nan', 341096000, 110), +('d5891efa-0e7d-409b-9bb7-b6ecc081c836', 960, 155, 2, 'nan', 522896000, 589), +('e4b960c8-33c6-4304-8355-9bafd70569ed', 113, 36, 2, 'nan', 330756000, 162), +('35c4eb46-3560-4d17-a210-979ba3037c53', 306, 75, 2, 'nan', 332952000, 93), +('2b85bd17-6d7e-48e0-8e02-477eb9ffe353', 263, 72, 2, 'nan', 328812000, 153), +('8c7e4562-dcc2-4a31-ab20-b873182b334d', 266, 67, 2, 'nan', 330156000, 124), +('49fda716-a469-4a6d-a3ef-3bcbd166e668', 155, 38, 2, 'nan', 331720000, 609), +('f424b26a-840a-42d3-a61f-3bbffd8b194d', 32, 38, 2, 'nan', 325684000, 216), +('7226d658-da67-4d7c-a898-e59cd77d977b', 1216, 88, 2, 'nan', 558752000, 625), +('50dc9288-e18c-41c2-a271-601289ff5685', 231, 101, 2, 'nan', 340116000, 158), +('4883db35-3ca6-4b0c-9377-cf7d1d97245d', 158, 85, 2, 'nan', 332072000, 1464), +('3ff09f4e-ada5-45d6-b7e1-fa003e31bf77', 122, 264, 2, 'nan', 337104000, 286), +('26c68022-6df2-4928-a4fb-220a156b86f7', 382, 94, 2, 'nan', 353068000, 200), +('cdd878a9-e3f1-4478-b613-90f2b2e9ccbd', 424, 201, 2, 'nan', 397292000, 647), +('d6e02749-45fb-4b92-8a8e-48b33b617f9c', 307, 80, 2, 'nan', 348920000, 206), +('47900ec7-0fea-41db-a320-c78b9da36538', 32, 80, 2, 'nan', 324068000, 240), +('1ef4d550-afaa-44b5-a06c-65286b9f3e00', 152, 80, 2, 'nan', 325820000, 278), +('8533d2a8-1fcf-4842-8826-c5458c002332', 31, 80, 2, 'nan', 319000000, 245), +('ab9095d1-af6d-49d5-9c87-bc2576f06135', 68, 43, 2, 'nan', 326896000, 235), +('61da73ff-b4ff-49a1-b775-c6215cfbd291', 231, 107, 2, 'nan', 333544000, 200), +('6c84dcf1-c5ea-4e69-b17f-d2d5b8d48bdf', 123, 50, 2, 'nan', 327520000, 82), +('dcb12603-4142-44d1-9a52-3ca3511e380e', 320, 44, 2, 'nan', 329448000, 475), +('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 41, 50, 2, 'nan', 301108000, 144); From d5996f14654c3ce5a85ec9a09e089038a0e00dd9 Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 7 Jan 2025 11:09:05 -0800 Subject: [PATCH 02/17] added colormap, create equation table --- qiita_db/meta_util.py | 24 ++++++------- qiita_db/support_files/patches/93.sql | 8 +++++ qiita_db/test/test_util.py | 17 ++++----- qiita_db/util.py | 52 ++++++++++++++++----------- qiita_pet/templates/resources.html | 47 ++++++++++++++++-------- 5 files changed, 90 insertions(+), 58 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index f4b9bf7ff..f394095f4 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -593,7 +593,7 @@ def update_resource_allocation_redis(active=True): if len(df) == 0: continue - fig, axs = resource_allocation_plot(df, cname, sname, col_name) + fig, axs = resource_allocation_plot(df, col_name) titles = [0, 0] images = [0, 0] @@ -605,21 +605,19 @@ def update_resource_allocation_redis(active=True): # only time new_fig = plt.figure() new_ax = new_fig.add_subplot(111) - - scatter_data = ax.collections[0] - new_ax.scatter(scatter_data.get_offsets()[:, 0], - scatter_data.get_offsets()[:, 1], - s=scatter_data.get_sizes(), label="data") - line = ax.lines[0] new_ax.plot(line.get_xdata(), line.get_ydata(), linewidth=1, color='orange') - - if len(ax.collections) > 1: - failure_data = ax.collections[1] - new_ax.scatter(failure_data.get_offsets()[:, 0], - failure_data.get_offsets()[:, 1], - color='red', s=3, label="failures") + + handles, labels = ax.get_legend_handles_labels() + for handle, label, scatter_data in zip(handles, + labels, + ax.collections): + color = handle.get_facecolor() + new_ax.scatter(scatter_data.get_offsets()[:, 0], + scatter_data.get_offsets()[:, 1], + s=scatter_data.get_sizes(), label=label, + color=color) new_ax.set_xscale('log') new_ax.set_yscale('log') diff --git a/qiita_db/support_files/patches/93.sql b/qiita_db/support_files/patches/93.sql index 4befc74d6..8fe8871bb 100644 --- a/qiita_db/support_files/patches/93.sql +++ b/qiita_db/support_files/patches/93.sql @@ -62,3 +62,11 @@ CREATE INDEX IF NOT EXISTS processing_job_command_parameters_payload ON qiita.pr -- Addding contraints for the slurm_reservation column ALTER TABLE qiita.analysis DROP CONSTRAINT IF EXISTS analysis_slurm_reservation_valid_chars; ALTER TABLE qiita.analysis ADD CONSTRAINT analysis_slurm_reservation_valid_chars CHECK ( slurm_reservation ~ '^[a-zA-Z0-9_]*$' ); + +-- Jan 7, 2025 +-- Adding a table for formulas for resource allocations +CREATE TABLE qiita.allocation_equations ( + equation_id SERIAL PRIMARY KEY, + equation_name TEXT NOT NULL, + expression TEXT NOT NULL + ); diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index cad800752..f449f15ba 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1327,8 +1327,7 @@ def setUp(self): def test_plot_return(self): # check the plot returns correct objects - fig1, axs1 = qdb.util.resource_allocation_plot( - self.df, self.cname, self.sname, self.col_name) + fig1, axs1 = qdb.util.resource_allocation_plot(self.df, self.col_name) self.assertIsInstance( fig1, Figure, "Returned object fig1 is not a Matplotlib Figure") @@ -1345,13 +1344,12 @@ def test_minimize_const(self): fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[0], self.cname, self.sname, 'MaxRSSRaw', - qdb.util.MODELS_MEM, self.col_name) + self.df, axs[0], 'MaxRSSRaw', qdb.util.MODELS_MEM, self.col_name) # check that the algorithm chooses correct model for MaxRSSRaw and # has 0 failures k, a, b = options.x - failures_df = qdb.util._resource_allocation_failures( - self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw') + failures_df = qdb.util._resource_allocation_success_failures( + self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')[-1] failures = failures_df.shape[0] self.assertEqual(bm, qdb.util.mem_model3, msg=f"""Best memory model @@ -1367,11 +1365,10 @@ def test_minimize_const(self): # check that the algorithm chooses correct model for ElapsedRaw and # has 1 failure bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[1], self.cname, self.sname, 'ElapsedRaw', - qdb.util.MODELS_TIME, self.col_name) + self.df, axs[1], 'ElapsedRaw', qdb.util.MODELS_TIME, self.col_name) k, a, b = options.x - failures_df = qdb.util._resource_allocation_failures( - self.df, k, a, b, bm, self.col_name, 'ElapsedRaw') + failures_df = qdb.util._resource_allocation_success_failures( + self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')[-1] failures = failures_df.shape[0] self.assertEqual(bm, qdb.util.time_model1, diff --git a/qiita_db/util.py b/qiita_db/util.py index c7346e15a..d92bd240c 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -74,6 +74,7 @@ from email.mime.text import MIMEText import matplotlib.pyplot as plt +from matplotlib import colormaps import numpy as np import pandas as pd from io import StringIO @@ -2363,17 +2364,13 @@ def send_email(to, subject, body): smtp.close() -def resource_allocation_plot(df, cname, sname, col_name): +def resource_allocation_plot(df, col_name): """Builds resource allocation plot for given filename and jobs Parameters ---------- file : str, required Builds plot for the specified file name. Usually provided as tsv.gz - cname: str, required - Specified job type - sname: str, required - Specified job sub type. col_name: str, required Specifies x axis for the graph @@ -2392,12 +2389,12 @@ def resource_allocation_plot(df, cname, sname, col_name): ax = axs[0] # models for memory _resource_allocation_plot_helper( - df, ax, cname, sname, "MaxRSSRaw", MODELS_MEM, col_name) + df, ax, "MaxRSSRaw", MODELS_MEM, col_name) ax = axs[1] # models for time _resource_allocation_plot_helper( - df, ax, cname, sname, "ElapsedRaw", MODELS_TIME, col_name) + df, ax, "ElapsedRaw", MODELS_TIME, col_name) return fig, axs @@ -2442,7 +2439,7 @@ def retrieve_resource_data(cname, sname, version, columns): def _resource_allocation_plot_helper( - df, ax, cname, sname, curr, models, col_name): + df, ax, curr, models, col_name): """Helper function for resource allocation plot. Builds plot for MaxRSSRaw and ElapsedRaw @@ -2459,14 +2456,14 @@ def _resource_allocation_plot_helper( col_name: str, required Specifies x axis for the graph curr: str, required - Either MaxRSSRaw or ElapsedRaw + Either MaxRSSRaw or ElapsedRaw (y axis) models: list, required List of functions that will be used for visualization """ x_data, y_data = df[col_name], df[curr] - ax.scatter(x_data, y_data, s=2, label="data") + # ax.scatter(x_data, y_data, s=2, label="data") d = dict() for index, row in df.iterrows(): x_value = row[col_name] @@ -2518,13 +2515,21 @@ def _resource_allocation_plot_helper( str(timedelta(seconds=round(cmin_value, 2))).rstrip('0').rstrip('.') x_plot = np.array(df[col_name]) - failures_df = _resource_allocation_failures( + success_df, failures_df = _resource_allocation_success_failures( df, k, a, b, best_model, col_name, curr) failures = failures_df.shape[0] - ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, label="failures") - + success_df['node_name'] = success_df['node_name'].fillna('unknown') + slurm_hosts = set(success_df['node_name'].tolist()) + cmap = colormaps.get_cmap('Accent').resampled(len(slurm_hosts)) + colors = [cmap( + i / (len(slurm_hosts) - 1)) for i in range(len(slurm_hosts))] + + for i, host in enumerate(slurm_hosts): + host_df = success_df[success_df['node_name'] == host] + ax.scatter(host_df[col_name], host_df[curr], color=colors[i], s=3, + label=host) ax.set_title( f'k||a||b: {k}||{a}||{b}\n' f'model: {get_model_name(best_model)}\n' @@ -2590,8 +2595,10 @@ def _resource_allocation_calculate( options = minimize(_resource_allocation_custom_loss, init, args=(x, y, model, middle)) k, a, b = options.x - failures_df = _resource_allocation_failures( - df, k, a, b, model, col_name, type_) + # important: here we take the 2nd (last) value of tuple since + # the helper function returns success, then failures. + failures_df = _resource_allocation_success_failures( + df, k, a, b, model, col_name, type_)[-1] y_plot = model(x, k, a, b) if not any(y_plot): continue @@ -2676,9 +2683,9 @@ def _resource_allocation_custom_loss(params, x, y, model, p): return np.mean(weighted_residuals) -def _resource_allocation_failures(df, k, a, b, model, col_name, type_): +def _resource_allocation_success_failures(df, k, a, b, model, col_name, type_): """Helper function for resource allocation plot. Creates a dataframe with - failures. + successes and failures given current model. Parameters ---------- @@ -2699,14 +2706,19 @@ def _resource_allocation_failures(df, k, a, b, model, col_name, type_): Returns ---------- - pandas.Dataframe - Dataframe containing failures for current type. + tuple with: + pandas.Dataframe + Dataframe containing successes for current type. + pandas.Dataframe + Dataframe containing failures for current type. """ x_plot = np.array(df[col_name]) df[f'c{type_}'] = model(x_plot, k, a, b) + success_df = df[df[type_] <= df[f'c{type_}']] failures_df = df[df[type_] > df[f'c{type_}']] - return failures_df + + return (success_df, failures_df) def MaxRSS_helper(x): diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html index e044429d2..988880e8c 100644 --- a/qiita_pet/templates/resources.html +++ b/qiita_pet/templates/resources.html @@ -114,19 +114,29 @@

Generated on: {{time}}

toggleDataVisibility(true); {% end %} - const commandsConst = JSON.parse(`{% raw commands %}`); - const softwareSelect = document.getElementById('software'); - const versionSelect = document.getElementById('version'); - const commandSelect = document.getElementById('command'); - - // Populate software options - for (const software in commandsConst) { - const option = document.createElement('option'); - option.value = software; - option.textContent = software; - softwareSelect.appendChild(option); - } + const commandsConst = JSON.parse(`{% raw commands %}`); + const softwareSelect = document.getElementById('software'); + const versionSelect = document.getElementById('version'); + const commandSelect = document.getElementById('command'); + + // Populate software options + for (const software in commandsConst) { + const option = document.createElement('option'); + option.value = software; + option.textContent = software; + softwareSelect.appendChild(option); + } + // If there's only one software option, select it automatically + function autoSelectIfSingle(selectElem) { + const realOptions = Array.from(selectElem.options).filter(opt => opt.value !== ""); + if (realOptions.length === 1) { + selectElem.value = realOptions[0].value; + // Trigger the change event to populate next select + const event = new Event('change', { bubbles: true }); + selectElem.dispatchEvent(event); + } + } function populateVersions(software) { versionSelect.innerHTML = ''; @@ -140,9 +150,10 @@

Generated on: {{time}}

versionSelect.appendChild(option); } } + // Auto-select if only one version available + autoSelectIfSingle(versionSelect); } - function populateCommands(software, version) { commandSelect.innerHTML = ''; @@ -154,6 +165,9 @@

Generated on: {{time}}

commandSelect.appendChild(option); }); } + + // Auto-select if only one command available + autoSelectIfSingle(commandSelect); } function sendPostRequest(software, version, command) { @@ -224,8 +238,7 @@

Generated on: {{time}}

} bootstrapAlert("Data updated successfully", "success", 2200); - } - else if (response.status === "no_data") { + } else if (response.status === "no_data") { toggleDataVisibility(false); $('#default-message').html('

No data available for the selected options.

'); bootstrapAlert("No data available", "info", 2200); @@ -264,5 +277,9 @@

Generated on: {{time}}

sendPostRequest(selectedSoftware, selectedVersion, selectedCommand); } }); + + // Attempt auto-select after initial population of software + autoSelectIfSingle(softwareSelect); + {% end %} From dd121439ae50ccf4bce2d76dcdf1a2ae4f064d1c Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 7 Jan 2025 11:29:44 -0800 Subject: [PATCH 03/17] Fixed styling --- qiita_db/meta_util.py | 1 - qiita_db/util.py | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index f394095f4..7c48b8493 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -608,7 +608,6 @@ def update_resource_allocation_redis(active=True): line = ax.lines[0] new_ax.plot(line.get_xdata(), line.get_ydata(), linewidth=1, color='orange') - handles, labels = ax.get_legend_handles_labels() for handle, label, scatter_data in zip(handles, labels, diff --git a/qiita_db/util.py b/qiita_db/util.py index d92bd240c..7df5642f1 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -94,8 +94,8 @@ time_model1 = (lambda x, k, a, b: a + b + np.log(x) * k) time_model2 = (lambda x, k, a, b: a + b * x + np.log(x) * k) time_model3 = (lambda x, k, a, b: a + b * np.log(x)**2 + np.log(x) * k) -time_model4 = (lambda x, k, a, b: a * np.log(x)**3 + b * np.log(x)**2 - + np.log(x) * k) +time_model4 = (lambda x, k, a, b: a * np.log(x)**3 + b * np.log(x)**2 + + np.log(x) * k) MODELS_TIME = [time_model1, time_model2, time_model3, time_model4] @@ -2717,7 +2717,6 @@ def _resource_allocation_success_failures(df, k, a, b, model, col_name, type_): df[f'c{type_}'] = model(x_plot, k, a, b) success_df = df[df[type_] <= df[f'c{type_}']] failures_df = df[df[type_] > df[f'c{type_}']] - return (success_df, failures_df) @@ -2846,8 +2845,8 @@ def update_resource_allocation_table(weeks=1, test=None): def merge_rows(rows): date_fmt = '%Y-%m-%dT%H:%M:%S' wait_time = ( - datetime.strptime(rows.iloc[0]['Start'], date_fmt) - - datetime.strptime(rows.iloc[0]['Submit'], date_fmt)) + datetime.strptime(rows.iloc[0]['Start'], date_fmt) - + datetime.strptime(rows.iloc[0]['Submit'], date_fmt)) if rows.shape[0] >= 2: tmp = rows.iloc[1].copy() else: From 1c30fc8c0ea437284bc122bb46974330e11b1a67 Mon Sep 17 00:00:00 2001 From: gossty Date: Thu, 9 Jan 2025 13:02:30 -0800 Subject: [PATCH 04/17] Using qiita.allocation_equations table in util.py --- .../support_files/patches/test_db_sql/93.sql | 9 ++ qiita_db/test/test_util.py | 29 +++-- qiita_db/util.py | 116 +++++++++++------- 3 files changed, 99 insertions(+), 55 deletions(-) create mode 100644 qiita_db/support_files/patches/test_db_sql/93.sql diff --git a/qiita_db/support_files/patches/test_db_sql/93.sql b/qiita_db/support_files/patches/test_db_sql/93.sql new file mode 100644 index 000000000..745caee48 --- /dev/null +++ b/qiita_db/support_files/patches/test_db_sql/93.sql @@ -0,0 +1,9 @@ +INSERT INTO qiita.allocation_equations(equation_name, expression) + VALUES ('mem_model1', 'k * np.log(x) + x * a + b'), +('mem_model2', 'k * np.log(x) + b * np.log(x)**2 + a'), +('mem_model3', 'k * np.log(x) + b * np.log(x)**2 + a * np.log(x)**3'), +('mem_model4', 'k * np.log(x) + b * np.log(x)**2 + a * np.log(x)**2.5'), +('time_model1', 'a + b + np.log(x) * k'), +('time_model2', 'a + b * x + np.log(x) * k'), +('time_model3', 'a + b * np.log(x)**2 + np.log(x) * k'), +('time_model4', 'a * np.log(x)**3 + b * np.log(x)**2 + np.log(x) * k'); diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index f449f15ba..a9d0868dc 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1343,42 +1343,45 @@ def test_minimize_const(self): self.df[self.col_name] = self.df.samples * self.df['columns'] fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) - bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[0], 'MaxRSSRaw', qdb.util.MODELS_MEM, self.col_name) + mem_models, time_models = qdb.util._retrieve_equations() + bm_name, bm, options = qdb.util._resource_allocation_plot_helper( + self.df, axs[0], 'MaxRSSRaw', mem_models, self.col_name) # check that the algorithm chooses correct model for MaxRSSRaw and # has 0 failures k, a, b = options.x failures_df = qdb.util._resource_allocation_success_failures( self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')[-1] failures = failures_df.shape[0] - self.assertEqual(bm, qdb.util.mem_model3, + + self.assertEqual(bm_name, 'mem_model3', + msg=f"""Best memory model + doesn't match + {bm_name} != 'mem_model3'""") + self.assertEqual(bm, mem_models['mem_model3'], msg=f"""Best memory model doesn't match Coefficients:{k} {a} {b} - {qdb.util.mem_model1}, "qdb.util.mem_model1" - {qdb.util.mem_model2}, "qdb.util.mem_model2" - {qdb.util.mem_model3}, "qdb.util.mem_model3" - {qdb.util.mem_model4}, "qdb.util.mem_model4" """) self.assertEqual(failures, 0, "Number of failures must be 0") # check that the algorithm chooses correct model for ElapsedRaw and # has 1 failure - bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[1], 'ElapsedRaw', qdb.util.MODELS_TIME, self.col_name) + bm_name, bm, options = qdb.util._resource_allocation_plot_helper( + self.df, axs[1], 'ElapsedRaw', time_models, self.col_name) k, a, b = options.x failures_df = qdb.util._resource_allocation_success_failures( self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')[-1] failures = failures_df.shape[0] + self.assertEqual(bm_name, 'time_model1', + msg=f"""Best time model + doesn't match + {bm_name} != 'time_model1'""") + self.assertEqual(bm, qdb.util.time_model1, msg=f"""Best time model doesn't match Coefficients:{k} {a} {b} - {qdb.util.time_model1}, "qdb.util.time_model1" - {qdb.util.time_model2}, "qdb.util.time_model2" - {qdb.util.time_model3}, "qdb.util.time_model3" - {qdb.util.time_model4}, "qdb.util.time_model4" """) self.assertEqual(failures, 1, "Number of failures must be 1") diff --git a/qiita_db/util.py b/qiita_db/util.py index 7df5642f1..a826c4c6c 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -81,41 +81,23 @@ from json import loads from scipy.optimize import minimize -# memory constant functions defined for @resource_allocation_plot -mem_model1 = (lambda x, k, a, b: k * np.log(x) + x * a + b) -mem_model2 = (lambda x, k, a, b: k * np.log(x) + b * np.log(x)**2 + a) -mem_model3 = (lambda x, k, a, b: k * np.log(x) + b * np.log(x)**2 + - a * np.log(x)**3) -mem_model4 = (lambda x, k, a, b: k * np.log(x) + b * np.log(x)**2 + - a * np.log(x)**2.5) -MODELS_MEM = [mem_model1, mem_model2, mem_model3, mem_model4] - -# time constant functions defined for @resource_allocation_plot -time_model1 = (lambda x, k, a, b: a + b + np.log(x) * k) -time_model2 = (lambda x, k, a, b: a + b * x + np.log(x) * k) -time_model3 = (lambda x, k, a, b: a + b * np.log(x)**2 + np.log(x) * k) -time_model4 = (lambda x, k, a, b: a * np.log(x)**3 + b * np.log(x)**2 + - np.log(x) * k) - -MODELS_TIME = [time_model1, time_model2, time_model3, time_model4] - def get_model_name(model): - if model == mem_model1: + if model == 'mem_model1': return "k * log(x) + x * a + b" - elif model == mem_model2: + elif model == 'mem_model2': return "k * log(x) + b * log(x)^2 + a" - elif model == mem_model3: + elif model == 'mem_model3': return "k * log(x) + b * log(x)^2 + a * log(x)^3" - elif model == mem_model4: + elif model == 'mem_model4': return "k * log(x) + b * log(x)^2 + a * log(x)^2.5" - elif model == time_model1: + elif model == 'time_model1': return "a + b + log(x) * k" - elif model == time_model2: + elif model == 'time_model2': return "a + b * x + log(x) * k" - elif model == time_model3: + elif model == 'time_model3': return "a + b * log(x)^2 + log(x) * k" - elif model == time_model4: + elif model == 'time_model4': return "a * log(x)^3 + b * log(x)^2 + log(x) * k" else: return "Unknown model" @@ -2387,19 +2369,63 @@ def resource_allocation_plot(df, col_name): fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) ax = axs[0] + mem_models, time_models = _retrieve_equations() + # models for memory _resource_allocation_plot_helper( - df, ax, "MaxRSSRaw", MODELS_MEM, col_name) - + df, ax, "MaxRSSRaw", mem_models, col_name) ax = axs[1] # models for time _resource_allocation_plot_helper( - df, ax, "ElapsedRaw", MODELS_TIME, col_name) + df, ax, "ElapsedRaw", time_models, col_name) return fig, axs +def _retrieve_equations(): + ''' + Helepr function for resource_allocation_plot. + Retrieves equations from db. Creates dictionary for memory and time models. + + Returns + ------- + tuple + dict + memory models - potential memory models for resource allocations + dict + time models - potential time models for resource allocations + ''' + memory_models = {} + time_models = {} + with qdb.sql_connection.TRN: + sql = ''' SELECT * FROM qiita.allocation_equations; ''' + qdb.sql_connection.TRN.add(sql) + res = qdb.sql_connection.TRN.execute_fetchindex() + for models in res: + if 'mem' in models[1]: + memory_models[models[1]] = lambda x, k, a, b: eval(models[2]) + else: + time_models[models[2]] = lambda x, k, a, b: eval(models[2]) + return (memory_models, time_models) + + def retrieve_resource_data(cname, sname, version, columns): + ''' + Retrieves resource data from db and constructs a DataFrame with relevant + fields. + + Parameters + ---------- + cname - command name for which we retrieve the resources + sname - software name for which we retrieve the resources + version - version of sftware for whhich we retrieve the resources + columns - column names for the DataFrame returned by this function + + Returns + ------- + pd.DataFrame + DataFrame with resources. + ''' with qdb.sql_connection.TRN: sql = """ SELECT @@ -2457,8 +2483,8 @@ def _resource_allocation_plot_helper( Specifies x axis for the graph curr: str, required Either MaxRSSRaw or ElapsedRaw (y axis) - models: list, required - List of functions that will be used for visualization + models: dictionary, required + Dictionary of functions that will be used for visualization """ @@ -2494,7 +2520,7 @@ def _resource_allocation_plot_helper( ax.set_xlabel(col_name) # 50 - number of maximum iterations, 3 - number of failures we tolerate - best_model, options = _resource_allocation_calculate( + best_model_name, best_model, options = _resource_allocation_calculate( df, x_data, y_data, models, curr, col_name, 50, 3) k, a, b = options.x x_plot = np.array(sorted(df[col_name].unique())) @@ -2522,9 +2548,11 @@ def _resource_allocation_plot_helper( label="failures") success_df['node_name'] = success_df['node_name'].fillna('unknown') slurm_hosts = set(success_df['node_name'].tolist()) - cmap = colormaps.get_cmap('Accent').resampled(len(slurm_hosts)) - colors = [cmap( - i / (len(slurm_hosts) - 1)) for i in range(len(slurm_hosts))] + cmap = colormaps.get_cmap('Accent') + if len(slurm_hosts) > len(cmap.colors): + raise ValueError(f"""'Accent' colormap only has {len(cmap.colors)} + colors, but {len(slurm_hosts)} hosts are provided.""") + colors = cmap.colors[:len(slurm_hosts)] for i, host in enumerate(slurm_hosts): host_df = success_df[success_df['node_name'] == host] @@ -2532,12 +2560,12 @@ def _resource_allocation_plot_helper( label=host) ax.set_title( f'k||a||b: {k}||{a}||{b}\n' - f'model: {get_model_name(best_model)}\n' + f'model: {get_model_name(best_model_name)}\n' f'real: {mini} || {maxi}\n' f'calculated: {cmin} || {cmax}\n' f'failures: {failures}') ax.legend(loc='upper left') - return best_model, options + return best_model_name, best_model, options def _resource_allocation_calculate( @@ -2555,8 +2583,8 @@ def _resource_allocation_calculate( current type (e.g. MaxRSSRaw) col_name: str, required Specifies x axis for the graph - models: list, required - List of functions that will be used for visualization + models: dictionary, required + Dictionary of functions that will be used for visualization depth: int, required Maximum number of iterations in binary search tolerance: int, required, @@ -2564,18 +2592,21 @@ def _resource_allocation_calculate( Returns ---------- + best_model_name: string + the name of the best model from the table best_model: function - best fitting function for the current list models + best fitting function for the current dictionary models best_result: object object containing constants for the best model (e.g. k, a, b in kx+b*a) """ init = [1, 1, 1] + best_model_name = None best_model = None best_result = None best_failures = np.inf best_max = np.inf - for model in models: + for model_name, model in models.items(): # start values for binary search, where sl is left, sr is right # penalty weight must be positive & non-zero, hence, sl >= 1. # the upper bound for error can be an arbitrary large number @@ -2646,9 +2677,10 @@ def _resource_allocation_calculate( if min_max <= best_max: best_failures = prev_failures best_max = min_max + best_model_name = model_name best_model = model best_result = res - return best_model, best_result + return best_model_name, best_model, best_result def _resource_allocation_custom_loss(params, x, y, model, p): From 82c95e1255ce984ffa0b868299b1cb8d5e62cb65 Mon Sep 17 00:00:00 2001 From: gossty Date: Thu, 9 Jan 2025 14:26:44 -0800 Subject: [PATCH 05/17] Debug --- .../support_files/patches/test_db_sql/93.sql | 17 +++++++++-------- qiita_db/test/test_meta_util.py | 9 +++++++-- qiita_db/test/test_util.py | 15 +++++++-------- qiita_db/util.py | 2 +- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/qiita_db/support_files/patches/test_db_sql/93.sql b/qiita_db/support_files/patches/test_db_sql/93.sql index 745caee48..bc87b5a7d 100644 --- a/qiita_db/support_files/patches/test_db_sql/93.sql +++ b/qiita_db/support_files/patches/test_db_sql/93.sql @@ -1,9 +1,10 @@ INSERT INTO qiita.allocation_equations(equation_name, expression) - VALUES ('mem_model1', 'k * np.log(x) + x * a + b'), -('mem_model2', 'k * np.log(x) + b * np.log(x)**2 + a'), -('mem_model3', 'k * np.log(x) + b * np.log(x)**2 + a * np.log(x)**3'), -('mem_model4', 'k * np.log(x) + b * np.log(x)**2 + a * np.log(x)**2.5'), -('time_model1', 'a + b + np.log(x) * k'), -('time_model2', 'a + b * x + np.log(x) * k'), -('time_model3', 'a + b * np.log(x)**2 + np.log(x) * k'), -('time_model4', 'a * np.log(x)**3 + b * np.log(x)**2 + np.log(x) * k'); + VALUES + ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), +('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), +('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**3))'), +('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), +('time_model1', 'a + b + ((np.log(x)) * k)'), +('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), +('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), +('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index d1e409a7d..ab10ebf76 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -532,13 +532,18 @@ def test_update_resource_allocation_redis(self): "model: " "k * log(x) + " "b * log(x)^2 + " - "a * log(x)^3" in title_mem + "a * log(x)^2.5" in title_mem ) title_time_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( cname, sname, version, col_name, 'title_time') title_time = str(r_client.get(title_time_str)) - self.assertTrue("model: a + b + log(x) * k" in title_time) + self.assertTrue( + "model: " + "a * log(x)^3 + " + "b * log(x)^2 + " + "log(x) * k" in title_time + ) if __name__ == '__main__': diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index a9d0868dc..ef07f62a7 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1353,11 +1353,11 @@ def test_minimize_const(self): self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')[-1] failures = failures_df.shape[0] - self.assertEqual(bm_name, 'mem_model3', + self.assertEqual(bm_name, 'mem_model4', msg=f"""Best memory model doesn't match - {bm_name} != 'mem_model3'""") - self.assertEqual(bm, mem_models['mem_model3'], + {bm_name} != 'mem_model4'""") + self.assertEqual(bm, mem_models['mem_model4'], msg=f"""Best memory model doesn't match Coefficients:{k} {a} {b} @@ -1372,18 +1372,17 @@ def test_minimize_const(self): failures_df = qdb.util._resource_allocation_success_failures( self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')[-1] failures = failures_df.shape[0] - - self.assertEqual(bm_name, 'time_model1', + self.assertEqual(bm_name, 'time_model4', msg=f"""Best time model doesn't match - {bm_name} != 'time_model1'""") + {bm_name} != 'time_model4'""") - self.assertEqual(bm, qdb.util.time_model1, + self.assertEqual(bm, time_models[bm_name], msg=f"""Best time model doesn't match Coefficients:{k} {a} {b} """) - self.assertEqual(failures, 1, "Number of failures must be 1") + self.assertEqual(failures, 0, "Number of failures must be 0") def test_MaxRSS_helper(self): tests = [ diff --git a/qiita_db/util.py b/qiita_db/util.py index a826c4c6c..3e8226bb7 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2405,7 +2405,7 @@ def _retrieve_equations(): if 'mem' in models[1]: memory_models[models[1]] = lambda x, k, a, b: eval(models[2]) else: - time_models[models[2]] = lambda x, k, a, b: eval(models[2]) + time_models[models[1]] = lambda x, k, a, b: eval(models[2]) return (memory_models, time_models) From 2cb3f37194215af93cd73abfc8c77ce3fbe17f20 Mon Sep 17 00:00:00 2001 From: gossty Date: Sun, 12 Jan 2025 13:33:42 -0800 Subject: [PATCH 06/17] Updates to @antgonza comments --- notebooks/resource-allocation/upload_df.py | 72 --------------- qiita_db/test/test_util.py | 6 +- qiita_db/util.py | 100 ++++++++++++--------- 3 files changed, 62 insertions(+), 116 deletions(-) delete mode 100644 notebooks/resource-allocation/upload_df.py diff --git a/notebooks/resource-allocation/upload_df.py b/notebooks/resource-allocation/upload_df.py deleted file mode 100644 index e74bcce04..000000000 --- a/notebooks/resource-allocation/upload_df.py +++ /dev/null @@ -1,72 +0,0 @@ -import pandas as pd - -# Example data loading -filename = './data/jobs_2024-02-21.tsv.gz' -df = pd.read_csv(filename, sep='\t', dtype={'extra_info': str}) - -# Convert string to timedelta, then to total seconds -df['ElapsedRawTime'] = pd.to_timedelta( - df['ElapsedRawTime']).apply( - lambda x: x.total_seconds()) - -cname = "Validate" -sname = "Diversity types - alpha_vector" -df = df[(df.cName == cname) & (df.sName == sname)] - -df['samples'] = df['samples'].fillna(0).astype(int) -df['columns'] = df['columns'].fillna(0).astype(int) -df['input_size'] = df['input_size'].fillna(0).astype(int) -df['MaxRSSRaw'] = df['MaxRSSRaw'].fillna(0).astype(int) -df['ElapsedRawTime'] = df['ElapsedRawTime'].fillna(0).astype(int) - -COL_NAME = 'samples * columns' -df[COL_NAME] = df['samples'] * df['columns'] -columns = ["MaxRSSRaw", "ElapsedRawTime"] -max_rows = [] - -for curr in columns: - # Get the maximum value for 'curr' within each COL_NAME group - max_values = df.groupby(COL_NAME)[curr].transform(max) - # Filter rows where the current column's value - # is the maximum within its group - curr_rows = df[df[curr] == max_values] - max_rows.append(curr_rows) - -filtered_df = pd.concat(max_rows).drop_duplicates().reset_index(drop=True) - -# INSERT INTO qiita.processing_job(processing_job_id, email, command_id, -# command_parameters, processing_job_status_id) -# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', -# 'admin@foo.com', 1, '""'::json, 1); - -# INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, -# columns, input_size, extra_info, memory_used, walltime_used) -# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', -# 327036000, 91); - -# processing_job_id uuid NOT NULL, -# samples integer, -# columns integer, -# input_size bigint, -# extra_info varchar DEFAULT NULL, -# memory_used bigint, -# walltime_used integer, - -res = "" - -for index, row in filtered_df.iterrows(): - res += f"""('{row['QiitaID']}', 'admin@foo.bar', 1, '""'::json, 1),\n""" -res += ";\n" -res += "Split\n" -for index, row in filtered_df.iterrows(): - res += ( - f"('{row['QiitaID']}', {int(row['samples'])}, " - f"{int(row['columns'])}, {int(row['input_size'])}, " - f"'{row['extra_info']}', {int(row['MaxRSSRaw'])}, " - f"{int(row['ElapsedRawTime'])}),\n" - ) - -res += ";\n" - -with open("sql.txt", 'w') as filename: - filename.write(res) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index ef07f62a7..0162af440 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1343,7 +1343,7 @@ def test_minimize_const(self): self.df[self.col_name] = self.df.samples * self.df['columns'] fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) - mem_models, time_models = qdb.util._retrieve_equations() + mem_models, time_models = qdb.util.retrieve_equations() bm_name, bm, options = qdb.util._resource_allocation_plot_helper( self.df, axs[0], 'MaxRSSRaw', mem_models, self.col_name) # check that the algorithm chooses correct model for MaxRSSRaw and @@ -1357,7 +1357,7 @@ def test_minimize_const(self): msg=f"""Best memory model doesn't match {bm_name} != 'mem_model4'""") - self.assertEqual(bm, mem_models['mem_model4'], + self.assertEqual(bm, mem_models['mem_model4']['equation'], msg=f"""Best memory model doesn't match Coefficients:{k} {a} {b} @@ -1377,7 +1377,7 @@ def test_minimize_const(self): doesn't match {bm_name} != 'time_model4'""") - self.assertEqual(bm, time_models[bm_name], + self.assertEqual(bm, time_models[bm_name]['equation'], msg=f"""Best time model doesn't match Coefficients:{k} {a} {b} diff --git a/qiita_db/util.py b/qiita_db/util.py index 3e8226bb7..8f3e83e15 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -82,27 +82,6 @@ from scipy.optimize import minimize -def get_model_name(model): - if model == 'mem_model1': - return "k * log(x) + x * a + b" - elif model == 'mem_model2': - return "k * log(x) + b * log(x)^2 + a" - elif model == 'mem_model3': - return "k * log(x) + b * log(x)^2 + a * log(x)^3" - elif model == 'mem_model4': - return "k * log(x) + b * log(x)^2 + a * log(x)^2.5" - elif model == 'time_model1': - return "a + b + log(x) * k" - elif model == 'time_model2': - return "a + b * x + log(x) * k" - elif model == 'time_model3': - return "a + b * log(x)^2 + log(x) * k" - elif model == 'time_model4': - return "a * log(x)^3 + b * log(x)^2 + log(x) * k" - else: - return "Unknown model" - - def scrub_data(s): r"""Scrubs data fields of characters not allowed by PostgreSQL @@ -2369,7 +2348,7 @@ def resource_allocation_plot(df, col_name): fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) ax = axs[0] - mem_models, time_models = _retrieve_equations() + mem_models, time_models = retrieve_equations() # models for memory _resource_allocation_plot_helper( @@ -2382,9 +2361,9 @@ def resource_allocation_plot(df, col_name): return fig, axs -def _retrieve_equations(): +def retrieve_equations(): ''' - Helepr function for resource_allocation_plot. + Helper function for resource_allocation_plot. Retrieves equations from db. Creates dictionary for memory and time models. Returns @@ -2397,16 +2376,40 @@ def _retrieve_equations(): ''' memory_models = {} time_models = {} + res = [] with qdb.sql_connection.TRN: sql = ''' SELECT * FROM qiita.allocation_equations; ''' qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() - for models in res: - if 'mem' in models[1]: - memory_models[models[1]] = lambda x, k, a, b: eval(models[2]) - else: - time_models[models[1]] = lambda x, k, a, b: eval(models[2]) - return (memory_models, time_models) + for models in res: + model_name = "Unknown model" + if models[1] == 'mem_model1': + model_name = "k * log(x) + x * a + b" + elif models[1] == 'mem_model2': + model_name = "k * log(x) + b * log(x)^2 + a" + elif models[1] == 'mem_model3': + model_name = "k * log(x) + b * log(x)^2 + a * log(x)^3" + elif models[1] == 'mem_model4': + model_name = "k * log(x) + b * log(x)^2 + a * log(x)^2.5" + elif models[1] == 'time_model1': + model_name = "a + b + log(x) * k" + elif models[1] == 'time_model2': + model_name = "a + b * x + log(x) * k" + elif models[1] == 'time_model3': + model_name = "a + b * log(x)^2 + log(x) * k" + elif models[1] == 'time_model4': + model_name = "a * log(x)^3 + b * log(x)^2 + log(x) * k" + if 'mem' in models[1]: + memory_models[models[1]] = { + "equation_name": model_name, + "equation": lambda x, k, a, b: eval(models[2]) + } + else: + time_models[models[1]] = { + "equation_name": model_name, + "equation": lambda x, k, a, b: eval(models[2]) + } + return (memory_models, time_models) def retrieve_resource_data(cname, sname, version, columns): @@ -2483,9 +2486,20 @@ def _resource_allocation_plot_helper( Specifies x axis for the graph curr: str, required Either MaxRSSRaw or ElapsedRaw (y axis) - models: dictionary, required - Dictionary of functions that will be used for visualization + models: dictionary, required. Follows this structure + equation_name: string + Human readable representation of the equation + equation: Python lambda function + Lambda function representing equation to optimizse + Returns + ------- + best_model_name: string + the name of the best model from the table + best_model: function + best fitting function for the current dictionary models + options: object + object containing constants for the best model (e.g. k, a, b in kx+b*a) """ x_data, y_data = df[col_name], df[curr] @@ -2560,7 +2574,7 @@ def _resource_allocation_plot_helper( label=host) ax.set_title( f'k||a||b: {k}||{a}||{b}\n' - f'model: {get_model_name(best_model_name)}\n' + f'model: {models[best_model_name]["equation_name"]}\n' f'real: {mini} || {maxi}\n' f'calculated: {cmin} || {cmax}\n' f'failures: {failures}') @@ -2583,8 +2597,11 @@ def _resource_allocation_calculate( current type (e.g. MaxRSSRaw) col_name: str, required Specifies x axis for the graph - models: dictionary, required - Dictionary of functions that will be used for visualization + models: dictionary, required. Follows this structure + equation_name: string + Human readable representation of the equation + equation: Python lambda function + Lambda function representing equation to optimizse depth: int, required Maximum number of iterations in binary search tolerance: int, required, @@ -2607,6 +2624,7 @@ def _resource_allocation_calculate( best_failures = np.inf best_max = np.inf for model_name, model in models.items(): + model_equation = model['equation'] # start values for binary search, where sl is left, sr is right # penalty weight must be positive & non-zero, hence, sl >= 1. # the upper bound for error can be an arbitrary large number @@ -2624,13 +2642,13 @@ def _resource_allocation_calculate( while left < right and cnt < depth: middle = (left + right) // 2 options = minimize(_resource_allocation_custom_loss, init, - args=(x, y, model, middle)) + args=(x, y, model_equation, middle)) k, a, b = options.x # important: here we take the 2nd (last) value of tuple since # the helper function returns success, then failures. failures_df = _resource_allocation_success_failures( - df, k, a, b, model, col_name, type_)[-1] - y_plot = model(x, k, a, b) + df, k, a, b, model_equation, col_name, type_)[-1] + y_plot = model_equation(x, k, a, b) if not any(y_plot): continue cmax = max(y_plot) @@ -2678,7 +2696,7 @@ def _resource_allocation_calculate( best_failures = prev_failures best_max = min_max best_model_name = model_name - best_model = model + best_model = model_equation best_result = res return best_model_name, best_model, best_result @@ -2695,8 +2713,8 @@ def _resource_allocation_custom_loss(params, x, y, model, p): Represents x data for the function calculation y: pandas.Series (pandas column), required Represents y data for the function calculation - models: list, required - List of functions that will be used for visualization + model: Python function + Lambda function representing current equation p: int, required Penalty weight for custom loss function From 2ea03e26f748f451774923d4ee7685185d13a1f8 Mon Sep 17 00:00:00 2001 From: gossty Date: Mon, 13 Jan 2025 18:40:28 -0800 Subject: [PATCH 07/17] Changes to @antgonza comments --- qiita_db/support_files/patches/93.sql | 8 -------- qiita_db/support_files/patches/94.sql | 7 +++++++ qiita_db/support_files/patches/test_db_sql/93.sql | 10 ---------- qiita_db/support_files/patches/test_db_sql/94.sql | 10 ++++++++++ qiita_db/util.py | 1 + 5 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 qiita_db/support_files/patches/94.sql delete mode 100644 qiita_db/support_files/patches/test_db_sql/93.sql create mode 100644 qiita_db/support_files/patches/test_db_sql/94.sql diff --git a/qiita_db/support_files/patches/93.sql b/qiita_db/support_files/patches/93.sql index 8fe8871bb..4befc74d6 100644 --- a/qiita_db/support_files/patches/93.sql +++ b/qiita_db/support_files/patches/93.sql @@ -62,11 +62,3 @@ CREATE INDEX IF NOT EXISTS processing_job_command_parameters_payload ON qiita.pr -- Addding contraints for the slurm_reservation column ALTER TABLE qiita.analysis DROP CONSTRAINT IF EXISTS analysis_slurm_reservation_valid_chars; ALTER TABLE qiita.analysis ADD CONSTRAINT analysis_slurm_reservation_valid_chars CHECK ( slurm_reservation ~ '^[a-zA-Z0-9_]*$' ); - --- Jan 7, 2025 --- Adding a table for formulas for resource allocations -CREATE TABLE qiita.allocation_equations ( - equation_id SERIAL PRIMARY KEY, - equation_name TEXT NOT NULL, - expression TEXT NOT NULL - ); diff --git a/qiita_db/support_files/patches/94.sql b/qiita_db/support_files/patches/94.sql new file mode 100644 index 000000000..3b565278b --- /dev/null +++ b/qiita_db/support_files/patches/94.sql @@ -0,0 +1,7 @@ +-- Jan 13, 2025 +-- Adding a table for formulas for resource allocations +CREATE TABLE qiita.allocation_equations ( + equation_id SERIAL PRIMARY KEY, + equation_name TEXT NOT NULL, + expression TEXT NOT NULL + ); \ No newline at end of file diff --git a/qiita_db/support_files/patches/test_db_sql/93.sql b/qiita_db/support_files/patches/test_db_sql/93.sql deleted file mode 100644 index bc87b5a7d..000000000 --- a/qiita_db/support_files/patches/test_db_sql/93.sql +++ /dev/null @@ -1,10 +0,0 @@ -INSERT INTO qiita.allocation_equations(equation_name, expression) - VALUES - ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), -('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), -('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**3))'), -('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), -('time_model1', 'a + b + ((np.log(x)) * k)'), -('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), -('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), -('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); diff --git a/qiita_db/support_files/patches/test_db_sql/94.sql b/qiita_db/support_files/patches/test_db_sql/94.sql new file mode 100644 index 000000000..99c85f61e --- /dev/null +++ b/qiita_db/support_files/patches/test_db_sql/94.sql @@ -0,0 +1,10 @@ +INSERT INTO qiita.allocation_equations(equation_name, expression) + VALUES + ('mem_model1', '(k * (log(x))) + (x * a) + b'), +('mem_model2', '(k * (log(x))) + (b * ((log(x))**2)) + a'), +('mem_model3', '(k * (log(x))) + (b * ((log(x))**2)) + (a * ((log(x))**3))'), +('mem_model4', '(k * (log(x))) + (b * ((log(x))**2)) + (a * ((log(x))**2.5))'), +('time_model1', 'a + b + ((log(x)) * k)'), +('time_model2', 'a + (b * x) + ((log(x)) * k)'), +('time_model3', 'a + (b * ((log(x))**2)) + ((log(x)) * k)'), +('time_model4', '(a * ((log(x))**3)) + (b * ((log(x))**2)) + ((log(x)) * k)'); diff --git a/qiita_db/util.py b/qiita_db/util.py index 8f3e83e15..15db31289 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -76,6 +76,7 @@ import matplotlib.pyplot as plt from matplotlib import colormaps import numpy as np +from numpy import log import pandas as pd from io import StringIO from json import loads From 60ec9fdcd9c5d24e5a730775ef124ddf9f51f195 Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 14 Jan 2025 10:20:04 -0800 Subject: [PATCH 08/17] Back to np.log --- .../support_files/patches/test_db_sql/94.sql | 16 +++++++------- qiita_db/test/test_meta_util.py | 14 +++++------- qiita_db/util.py | 22 ++----------------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/qiita_db/support_files/patches/test_db_sql/94.sql b/qiita_db/support_files/patches/test_db_sql/94.sql index 99c85f61e..41ec7d8a7 100644 --- a/qiita_db/support_files/patches/test_db_sql/94.sql +++ b/qiita_db/support_files/patches/test_db_sql/94.sql @@ -1,10 +1,10 @@ INSERT INTO qiita.allocation_equations(equation_name, expression) VALUES - ('mem_model1', '(k * (log(x))) + (x * a) + b'), -('mem_model2', '(k * (log(x))) + (b * ((log(x))**2)) + a'), -('mem_model3', '(k * (log(x))) + (b * ((log(x))**2)) + (a * ((log(x))**3))'), -('mem_model4', '(k * (log(x))) + (b * ((log(x))**2)) + (a * ((log(x))**2.5))'), -('time_model1', 'a + b + ((log(x)) * k)'), -('time_model2', 'a + (b * x) + ((log(x)) * k)'), -('time_model3', 'a + (b * ((log(x))**2)) + ((log(x)) * k)'), -('time_model4', '(a * ((log(x))**3)) + (b * ((log(x))**2)) + ((log(x)) * k)'); + ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), +('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), +('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'), +('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), +('time_model1', 'a + b + ((np.log(x)) * k)'), +('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), +('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), +('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index ab10ebf76..fdf55d101 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -529,20 +529,18 @@ def test_update_resource_allocation_redis(self): cname, sname, version, col_name, 'title_mem') title_mem = str(r_client.get(title_mem_str)) self.assertTrue( - "model: " - "k * log(x) + " - "b * log(x)^2 + " - "a * log(x)^2.5" in title_mem + "model: (k * (np.log(x))) + " + "(b * ((np.log(x))**2)) + " + "(a * ((np.log(x))**2.5))" in title_mem ) title_time_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( cname, sname, version, col_name, 'title_time') title_time = str(r_client.get(title_time_str)) self.assertTrue( - "model: " - "a * log(x)^3 + " - "b * log(x)^2 + " - "log(x) * k" in title_time + "model: (a * ((np.log(x))**3)) + " + "(b * ((np.log(x))**2)) + " + "((np.log(x)) * k)" in title_time ) diff --git a/qiita_db/util.py b/qiita_db/util.py index 15db31289..16b9f9ea2 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -76,7 +76,6 @@ import matplotlib.pyplot as plt from matplotlib import colormaps import numpy as np -from numpy import log import pandas as pd from io import StringIO from json import loads @@ -2383,31 +2382,14 @@ def retrieve_equations(): qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() for models in res: - model_name = "Unknown model" - if models[1] == 'mem_model1': - model_name = "k * log(x) + x * a + b" - elif models[1] == 'mem_model2': - model_name = "k * log(x) + b * log(x)^2 + a" - elif models[1] == 'mem_model3': - model_name = "k * log(x) + b * log(x)^2 + a * log(x)^3" - elif models[1] == 'mem_model4': - model_name = "k * log(x) + b * log(x)^2 + a * log(x)^2.5" - elif models[1] == 'time_model1': - model_name = "a + b + log(x) * k" - elif models[1] == 'time_model2': - model_name = "a + b * x + log(x) * k" - elif models[1] == 'time_model3': - model_name = "a + b * log(x)^2 + log(x) * k" - elif models[1] == 'time_model4': - model_name = "a * log(x)^3 + b * log(x)^2 + log(x) * k" if 'mem' in models[1]: memory_models[models[1]] = { - "equation_name": model_name, + "equation_name": models[2], "equation": lambda x, k, a, b: eval(models[2]) } else: time_models[models[1]] = { - "equation_name": model_name, + "equation_name": models[2], "equation": lambda x, k, a, b: eval(models[2]) } return (memory_models, time_models) From 49b4a794a13a9271d91017fd9c41e5e4dc8ba11d Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 11 Feb 2025 11:29:01 -0800 Subject: [PATCH 09/17] Added column_name variability --- qiita_db/meta_util.py | 193 ++++++++++++------ qiita_db/support_files/patches/94.sql | 30 ++- .../support_files/patches/test_db_sql/94.sql | 10 - qiita_db/test/test_meta_util.py | 2 +- qiita_db/test/test_util.py | 16 +- qiita_db/util.py | 48 +++-- qiita_pet/handlers/resources.py | 9 +- qiita_pet/templates/resources.html | 54 ++++- 8 files changed, 241 insertions(+), 121 deletions(-) delete mode 100644 qiita_db/support_files/patches/test_db_sql/94.sql diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 7c48b8493..c76508f7a 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -555,20 +555,45 @@ def generate_plugin_releases(): f(redis_key, v) -def get_software_commands(active): +def _build_software_commands_col_names_object(active): + ''' + Helper function for update_resource_allocation_redis + + Gets all the available software with the corresponding software versions, + with the corresponding commands, with the corresponding col_names, with + column names from db + ''' + + col_name_list = [] + with qdb.sql_connection.TRN: + sql = ''' SELECT * FROM qiita.resource_allocation_column_names; ''' + qdb.sql_connection.TRN.add(sql) + res = qdb.sql_connection.TRN.execute_fetchindex() + + for col_name in res: + col_name_list.append(col_name[1]) + software_list = [s for s in qdb.software.Software.iter(active=active)] - software_commands = defaultdict(lambda: defaultdict(list)) + software_commands = defaultdict(lambda: defaultdict( + lambda: defaultdict(list))) for software in software_list: sname = software.name sversion = software.version - commands = software.commands - for command in commands: - software_commands[sname][sversion].append(command.name) - software_commands[sname] = dict(software_commands[sname]) + for command in software.commands: + cmd_name = command.name + for col in col_name_list: + software_commands[sname][sversion][cmd_name].append(col) - return dict(software_commands) + final_obj = { + sname: { + sversion: dict(commands) + for sversion, commands in dict(versions).items() + } + for sname, versions in dict(software_commands).items() + } + return final_obj def update_resource_allocation_redis(active=True): @@ -581,70 +606,106 @@ def update_resource_allocation_redis(active=True): """ time = datetime.now().strftime('%m-%d-%y') - scommands = get_software_commands(active) + + scommands = _build_software_commands_col_names_object(active) + redis_key = 'resources:commands' r_client.set(redis_key, str(scommands)) for sname, versions in scommands.items(): for version, commands in versions.items(): - for cname in commands: - col_name = "samples * columns" + for cname, col_name_list in commands.items(): df = retrieve_resource_data(cname, sname, version, COLUMNS) + print(("Retrieving allocation resources for " + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}")) if len(df) == 0: + print(("No allocation resources available for" + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}")) continue - - fig, axs = resource_allocation_plot(df, col_name) - titles = [0, 0] - images = [0, 0] - - # Splitting 1 image plot into 2 separate for better layout. - for i, ax in enumerate(axs): - titles[i] = ax.get_title() - ax.set_title("") - # new_fig, new_ax – copy with either only memory plot or - # only time - new_fig = plt.figure() - new_ax = new_fig.add_subplot(111) - line = ax.lines[0] - new_ax.plot(line.get_xdata(), line.get_ydata(), - linewidth=1, color='orange') - handles, labels = ax.get_legend_handles_labels() - for handle, label, scatter_data in zip(handles, - labels, - ax.collections): - color = handle.get_facecolor() - new_ax.scatter(scatter_data.get_offsets()[:, 0], - scatter_data.get_offsets()[:, 1], - s=scatter_data.get_sizes(), label=label, - color=color) - - new_ax.set_xscale('log') - new_ax.set_yscale('log') - new_ax.set_xlabel(ax.get_xlabel()) - new_ax.set_ylabel(ax.get_ylabel()) - new_ax.legend(loc='upper left') - - new_fig.tight_layout() - plot = BytesIO() - new_fig.savefig(plot, format='png') - plot.seek(0) - img = 'data:image/png;base64,' + quote( - b64encode(plot.getvalue()).decode('ascii')) - images[i] = img - plt.close(new_fig) - plt.close(fig) - - # SID, CID, col_name - values = [ - ("img_mem", images[0], r_client.set), - ("img_time", images[1], r_client.set), - ('time', time, r_client.set), - ("title_mem", titles[0], r_client.set), - ("title_time", titles[1], r_client.set) - ] - - for k, v, f in values: - redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( - cname, sname, version, col_name, k) - r_client.delete(redis_key) - f(redis_key, v) + # column_name_str looks like col1*col2*col3, etc + for col_name_str in col_name_list: + df_copy = df.copy() + new_column = None + col_name_split = col_name_str.split('*') + df_copy.dropna(subset=col_name_split, inplace=True) + + # Create a column with the desired columns + for curr_column in col_name_split: + if new_column is None: + new_column = df_copy[curr_column] + else: + new_column *= df_copy[curr_column] + print(("Building resource allocation plot for " + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) + + fig, axs = resource_allocation_plot(df_copy, + col_name_str, + new_column) + titles = [0, 0] + images = [0, 0] + + # Splitting 1 image plot into 2 separate for better layout. + for i, ax in enumerate(axs): + titles[i] = ax.get_title() + ax.set_title("") + # new_fig, new_ax – copy with either only memory plot + # or only time + new_fig = plt.figure() + new_ax = new_fig.add_subplot(111) + line = ax.lines[0] + new_ax.plot(line.get_xdata(), line.get_ydata(), + linewidth=1, color='orange') + handles, labels = ax.get_legend_handles_labels() + for handle, label, scatter_data in zip( + handles, + labels, + ax.collections): + color = handle.get_facecolor() + new_ax.scatter(scatter_data.get_offsets()[:, 0], + scatter_data.get_offsets()[:, 1], + s=scatter_data.get_sizes(), + label=label, + color=color) + + new_ax.set_xscale('log') + new_ax.set_yscale('log') + new_ax.set_xlabel(ax.get_xlabel()) + new_ax.set_ylabel(ax.get_ylabel()) + new_ax.legend(loc='upper left') + + new_fig.tight_layout() + plot = BytesIO() + new_fig.savefig(plot, format='png') + plot.seek(0) + img = 'data:image/png;base64,' + quote( + b64encode(plot.getvalue()).decode('ascii')) + images[i] = img + plt.close(new_fig) + plt.close(fig) + + # SID, CID, col_name + values = [ + ("img_mem", images[0], r_client.set), + ("img_time", images[1], r_client.set), + ('time', time, r_client.set), + ("title_mem", titles[0], r_client.set), + ("title_time", titles[1], r_client.set) + ] + print(("Saving resource allocation image for " + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) + + for k, v, f in values: + redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( + cname, sname, version, col_name_str, k) + r_client.delete(redis_key) + f(redis_key, v) diff --git a/qiita_db/support_files/patches/94.sql b/qiita_db/support_files/patches/94.sql index 3b565278b..428f76544 100644 --- a/qiita_db/support_files/patches/94.sql +++ b/qiita_db/support_files/patches/94.sql @@ -1,7 +1,27 @@ -- Jan 13, 2025 -- Adding a table for formulas for resource allocations -CREATE TABLE qiita.allocation_equations ( - equation_id SERIAL PRIMARY KEY, - equation_name TEXT NOT NULL, - expression TEXT NOT NULL - ); \ No newline at end of file +CREATE TABLE qiita.resource_allocation_equations ( + equation_id SERIAL PRIMARY KEY, + equation_name TEXT NOT NULL, + expression TEXT NOT NULL + ); + +INSERT INTO qiita.resource_allocation_equations(equation_name, expression) VALUES + ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), + ('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), + ('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'), + ('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), + ('time_model1', 'a + b + ((np.log(x)) * k)'), + ('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), + ('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), + ('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); + +CREATE TABLE qiita.resource_allocation_column_names ( + col_name_id SERIAL PRIMARY KEY, + col_name TEXT NOT NULL + ); + +INSERT INTO qiita.resource_allocation_column_names(col_name) VALUES + ('samples'), ('columns'), ('input_size'), + ('samples*columns'), ('samples*input_size'), + ('columns*input_size'), ('samples*columns*input_size'); \ No newline at end of file diff --git a/qiita_db/support_files/patches/test_db_sql/94.sql b/qiita_db/support_files/patches/test_db_sql/94.sql deleted file mode 100644 index 41ec7d8a7..000000000 --- a/qiita_db/support_files/patches/test_db_sql/94.sql +++ /dev/null @@ -1,10 +0,0 @@ -INSERT INTO qiita.allocation_equations(equation_name, expression) - VALUES - ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), -('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), -('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'), -('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), -('time_model1', 'a + b + ((np.log(x)) * k)'), -('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), -('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), -('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index fdf55d101..7503d3c4b 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -522,7 +522,7 @@ def test_generate_plugin_releases(self): def test_update_resource_allocation_redis(self): cname = "Split libraries FASTQ" sname = "QIIMEq2" - col_name = "samples * columns" + col_name = "samples*columns" version = "1.9.1" qdb.meta_util.update_resource_allocation_redis(False) title_mem_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 0162af440..7838fcd26 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1314,7 +1314,7 @@ def setUp(self): self.cname = "Split libraries FASTQ" self.sname = "QIIMEq2" self.version = "1.9.1" - self.col_name = 'samples * columns' + self.col_name = 'samples*columns' self.columns = [ "sName", "sVersion", "cID", "cName", "processing_job_id", "parameters", "samples", "columns", "input_size", "extra_info", @@ -1325,9 +1325,13 @@ def setUp(self): self.df = qdb.util.retrieve_resource_data( self.cname, self.sname, self.version, self.columns) + self.df.dropna(subset=['samples', 'columns'], inplace=True) + self.df[self.col_name] = self.df.samples * self.df['columns'] + def test_plot_return(self): # check the plot returns correct objects - fig1, axs1 = qdb.util.resource_allocation_plot(self.df, self.col_name) + fig1, axs1 = qdb.util.resource_allocation_plot(self.df, self.col_name, + self.df[self.col_name]) self.assertIsInstance( fig1, Figure, "Returned object fig1 is not a Matplotlib Figure") @@ -1337,13 +1341,10 @@ def test_plot_return(self): "Returned object axs1 is not a single Matplotlib Axes object") def test_minimize_const(self): - self.df = self.df[ - (self.df.cName == self.cname) & (self.df.sName == self.sname)] - self.df.dropna(subset=['samples', 'columns'], inplace=True) - self.df[self.col_name] = self.df.samples * self.df['columns'] + fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) - mem_models, time_models = qdb.util.retrieve_equations() + mem_models, time_models = qdb.util._retrieve_equations() bm_name, bm, options = qdb.util._resource_allocation_plot_helper( self.df, axs[0], 'MaxRSSRaw', mem_models, self.col_name) # check that the algorithm chooses correct model for MaxRSSRaw and @@ -1418,7 +1419,6 @@ def test_db_update(self): '8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0' ] } - qdb.util.update_resource_allocation_table(test=test_data) for curr_cname, ids in types.items(): diff --git a/qiita_db/util.py b/qiita_db/util.py index 16b9f9ea2..349143c15 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2325,15 +2325,17 @@ def send_email(to, subject, body): smtp.close() -def resource_allocation_plot(df, col_name): +def resource_allocation_plot(df, col_name_str, curr_column): """Builds resource allocation plot for given filename and jobs Parameters ---------- - file : str, required - Builds plot for the specified file name. Usually provided as tsv.gz - col_name: str, required - Specifies x axis for the graph + df : pd.Dataframe, required + Builds plot for the specified dataframe. + col_name_str: str, required + Column name for the x axis that will be used to build the plots. + curr_column: pd.Series, requirew + Pandas Series representing a column with col_name_str. Returns ---------- @@ -2341,27 +2343,26 @@ def resource_allocation_plot(df, col_name): Returns a matplotlib object with a plot """ - df.dropna(subset=['samples', 'columns'], inplace=True) - df[col_name] = df.samples * df['columns'] - df[col_name] = df[col_name].astype(int) - fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) - ax = axs[0] - mem_models, time_models = retrieve_equations() + + mem_models, time_models = _retrieve_equations() + df[col_name_str] = curr_column # models for memory + print("\tCalculating best model for memory") _resource_allocation_plot_helper( - df, ax, "MaxRSSRaw", mem_models, col_name) + df, ax, "MaxRSSRaw", mem_models, col_name_str) ax = axs[1] + # models for time + print("\tCalculating best model for time") _resource_allocation_plot_helper( - df, ax, "ElapsedRaw", time_models, col_name) - + df, ax, "ElapsedRaw", time_models, col_name_str) return fig, axs -def retrieve_equations(): +def _retrieve_equations(): ''' Helper function for resource_allocation_plot. Retrieves equations from db. Creates dictionary for memory and time models. @@ -2378,7 +2379,7 @@ def retrieve_equations(): time_models = {} res = [] with qdb.sql_connection.TRN: - sql = ''' SELECT * FROM qiita.allocation_equations; ''' + sql = ''' SELECT * FROM qiita.resource_allocation_equations; ''' qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() for models in res: @@ -2516,9 +2517,15 @@ def _resource_allocation_plot_helper( ax.set_ylabel(curr) ax.set_xlabel(col_name) + print(f"\t\tFitting best model for {curr}; column {col_name}") # 50 - number of maximum iterations, 3 - number of failures we tolerate best_model_name, best_model, options = _resource_allocation_calculate( df, x_data, y_data, models, curr, col_name, 50, 3) + if options is None: + print(df) + print(x_data) + print(y_data) + print(f"\t\tSuccessfully chose best model for {curr}; column {col_name}") k, a, b = options.x x_plot = np.array(sorted(df[col_name].unique())) y_plot = best_model(x_plot, k, a, b) @@ -2543,9 +2550,10 @@ def _resource_allocation_plot_helper( failures = failures_df.shape[0] ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, label="failures") - success_df['node_name'] = success_df['node_name'].fillna('unknown') + + success_df.loc[:, 'node_name'] = success_df['node_name'].fillna('unknown') slurm_hosts = set(success_df['node_name'].tolist()) - cmap = colormaps.get_cmap('Accent') + cmap = colormaps['Accent'] if len(slurm_hosts) > len(cmap.colors): raise ValueError(f"""'Accent' colormap only has {len(cmap.colors)} colors, but {len(slurm_hosts)} hosts are provided.""") @@ -2627,7 +2635,7 @@ def _resource_allocation_calculate( options = minimize(_resource_allocation_custom_loss, init, args=(x, y, model_equation, middle)) k, a, b = options.x - # important: here we take the 2nd (last) value of tuple since + # IMPORTANT: here we take the 2nd (last) value of tuple since # the helper function returns success, then failures. failures_df = _resource_allocation_success_failures( df, k, a, b, model_equation, col_name, type_)[-1] @@ -2681,6 +2689,8 @@ def _resource_allocation_calculate( best_model_name = model_name best_model = model_equation best_result = res + elif best_result is None: + best_result = res return best_model_name, best_model, best_result diff --git a/qiita_pet/handlers/resources.py b/qiita_pet/handlers/resources.py index 0b5873997..62d9e338f 100644 --- a/qiita_pet/handlers/resources.py +++ b/qiita_pet/handlers/resources.py @@ -6,8 +6,7 @@ from qiita_core.qiita_settings import r_client from qiita_core.util import execute_as_transaction -commands = 'resources:commands' -default_col_name = "samples * columns" +COMMANDS = 'resources:commands' class ResourcesHandler(BaseHandler): @@ -34,7 +33,7 @@ def _get_resources(self, cname, sname, version, col_name, callback): @execute_as_transaction def _get_commands(self, callback): - res = r_client.get(commands) + res = r_client.get(COMMANDS) callback(res) @authenticated @@ -47,7 +46,6 @@ def get(self): commands_str = commands.decode('utf-8') commands_dict = ast.literal_eval(commands_str) commands_json = json.dumps(commands_dict) - self.render('resources.html', img_mem=None, img_time=None, time=None, @@ -69,9 +67,10 @@ def post(self): software = data.get('software') version = data.get('version') command = data.get('command') + col_name = data.get('col_name') resources = yield Task(self._get_resources, command, software, - version, default_col_name) + version, col_name) mcof, mmodel, mreal, mcalc, mfail = list( map(lambda x: x.split(b": ")[1].strip().decode('utf-8'), diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html index 988880e8c..85b2e0ac9 100644 --- a/qiita_pet/templates/resources.html +++ b/qiita_pet/templates/resources.html @@ -35,6 +35,14 @@

Please choose software, version, and command to view the data.

+ +
+ + +
+
@@ -118,6 +126,7 @@

Generated on: {{time}}

const softwareSelect = document.getElementById('software'); const versionSelect = document.getElementById('version'); const commandSelect = document.getElementById('command'); + const colNameSelect = document.getElementById('col_name'); // Populate software options for (const software in commandsConst) { @@ -141,6 +150,7 @@

Generated on: {{time}}

function populateVersions(software) { versionSelect.innerHTML = ''; commandSelect.innerHTML = ''; + colNameSelect.innerHTML = ''; if (software && commandsConst[software]) { for (const version in commandsConst[software]) { @@ -156,25 +166,46 @@

Generated on: {{time}}

function populateCommands(software, version) { commandSelect.innerHTML = ''; + colNameSelect.innerHTML = ''; + console.log(software, version); + console.log(commandsConst[software][version]); if (software && version && commandsConst[software][version]) { - commandsConst[software][version].forEach(command => { + for (const command in commandsConst[software][version]) { const option = document.createElement('option'); option.value = command; option.textContent = command; commandSelect.appendChild(option); - }); + } } // Auto-select if only one command available autoSelectIfSingle(commandSelect); } - function sendPostRequest(software, version, command) { + function populateColNames(software, version, command) { + colNameSelect.innerHTML = ''; + + if (software && version && command && commandsConst[software][version][command]) { + commandsConst[software][version][command].forEach(colName => { + const option = document.createElement('option'); + option.value = colName; + option.textContent = colName; + colNameSelect.appendChild(option); + }); + } + + // Auto-select if only one command available + autoSelectIfSingle(colNameSelect); + } + + function sendPostRequest(software, version, command, colName) { + const data = { software: software, version: version, - command: command + command: command, + col_name: colName }; $.post(window.location.href, JSON.stringify(data), function(response, textStatus, jqXHR) { @@ -254,6 +285,7 @@

Generated on: {{time}}

} + // Event listener for software select softwareSelect.addEventListener('change', function() { const selectedSoftware = this.value; @@ -272,10 +304,18 @@

Generated on: {{time}}

const selectedSoftware = softwareSelect.value; const selectedVersion = versionSelect.value; const selectedCommand = this.value; + populateColNames(selectedSoftware, selectedVersion, selectedCommand); + }); - if (selectedSoftware && selectedVersion && selectedCommand) { - sendPostRequest(selectedSoftware, selectedVersion, selectedCommand); - } + colNameSelect.addEventListener('change', function() { + const selectedSoftware = softwareSelect.value; + const selectedVersion = versionSelect.value; + const selectedCommand = commandSelect.value; + const selectedColumn = this.value; + + if (selectedSoftware && selectedVersion && selectedCommand && selectedColumn) { + sendPostRequest(selectedSoftware, selectedVersion, selectedCommand, selectedColumn); + } }); // Attempt auto-select after initial population of software From fbe01a2cb685d1b0c560015e1eed7f01ac5123c3 Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 11 Feb 2025 11:31:30 -0800 Subject: [PATCH 10/17] Fix styling --- qiita_db/meta_util.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index c76508f7a..9254ce9f6 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -622,9 +622,9 @@ def update_resource_allocation_redis(active=True): f" command: {cname}")) if len(df) == 0: print(("No allocation resources available for" + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}")) + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}")) continue # column_name_str looks like col1*col2*col3, etc for col_name_str in col_name_list: @@ -640,10 +640,10 @@ def update_resource_allocation_redis(active=True): else: new_column *= df_copy[curr_column] print(("Building resource allocation plot for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name_str}")) + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) fig, axs = resource_allocation_plot(df_copy, col_name_str, @@ -699,10 +699,10 @@ def update_resource_allocation_redis(active=True): ("title_time", titles[1], r_client.set) ] print(("Saving resource allocation image for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name_str}")) + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) for k, v, f in values: redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( From 9717adb469625ae6d86944ad71e56d3ef94d55ef Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 11 Feb 2025 12:23:10 -0800 Subject: [PATCH 11/17] Update description of software retrieval function --- qiita_db/meta_util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 9254ce9f6..78cfbd514 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -559,9 +559,8 @@ def _build_software_commands_col_names_object(active): ''' Helper function for update_resource_allocation_redis - Gets all the available software with the corresponding software versions, - with the corresponding commands, with the corresponding col_names, with - column names from db + Constructs a nested dictionary containing software commands and associated + column names from the database. ''' col_name_list = [] From 8d27f890a5e86cad685c65dfb9b2c0a62bce872e Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 11 Feb 2025 12:37:58 -0800 Subject: [PATCH 12/17] Remove the unnecessary test_db 94.sql file --- qiita_db/meta_util.py | 2 +- qiita_db/support_files/patches/test_db_sql/94.sql | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 qiita_db/support_files/patches/test_db_sql/94.sql diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 78cfbd514..e172bcb86 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -559,7 +559,7 @@ def _build_software_commands_col_names_object(active): ''' Helper function for update_resource_allocation_redis - Constructs a nested dictionary containing software commands and associated + Constructs a nested dictionary containing software commands and associated column names from the database. ''' diff --git a/qiita_db/support_files/patches/test_db_sql/94.sql b/qiita_db/support_files/patches/test_db_sql/94.sql deleted file mode 100644 index 41ec7d8a7..000000000 --- a/qiita_db/support_files/patches/test_db_sql/94.sql +++ /dev/null @@ -1,10 +0,0 @@ -INSERT INTO qiita.allocation_equations(equation_name, expression) - VALUES - ('mem_model1', '(k * (np.log(x))) + (x * a) + b'), -('mem_model2', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + a'), -('mem_model3', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.np.log(x))**3))'), -('mem_model4', '(k * (np.log(x))) + (b * ((np.log(x))**2)) + (a * ((np.log(x))**2.5))'), -('time_model1', 'a + b + ((np.log(x)) * k)'), -('time_model2', 'a + (b * x) + ((np.log(x)) * k)'), -('time_model3', 'a + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'), -('time_model4', '(a * ((np.log(x))**3)) + (b * ((np.log(x))**2)) + ((np.log(x)) * k)'); From 7fb202a130dca5048b6b3e05c6b2e5940dcb8cb1 Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 11 Feb 2025 13:15:35 -0800 Subject: [PATCH 13/17] Update test_user.py --- qiita_db/test/test_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/test/test_user.py b/qiita_db/test/test_user.py index 0d14d2c8a..20a90f3e9 100644 --- a/qiita_db/test/test_user.py +++ b/qiita_db/test/test_user.py @@ -482,7 +482,7 @@ def test_mark_messages(self): user.mark_messages([1, 2]) obs = user.messages() exp = [True, True, False] - self.assertEqual([x[3] for x in obs], exp) + self.assertCountEqual([x[3] for x in obs], exp) user.mark_messages([1], read=False) obs = user.messages() From c9ce88b98a7181859c186096c4daeef462a90e01 Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 12 Feb 2025 10:02:13 -0800 Subject: [PATCH 14/17] Updates to @antgonza comments --- qiita_db/meta_util.py | 108 +++++++++++++++++++----------------------- qiita_db/util.py | 45 ++++++++++-------- 2 files changed, 73 insertions(+), 80 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index e172bcb86..5304e976e 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -555,58 +555,42 @@ def generate_plugin_releases(): f(redis_key, v) -def _build_software_commands_col_names_object(active): - ''' - Helper function for update_resource_allocation_redis +def update_resource_allocation_redis(active=True, verbose=False): + """Updates redis with plots and information about current software. - Constructs a nested dictionary containing software commands and associated - column names from the database. - ''' + Parameters + ---------- + active: boolean, optional + Defaults to True. Should only be False when testing. - col_name_list = [] - with qdb.sql_connection.TRN: - sql = ''' SELECT * FROM qiita.resource_allocation_column_names; ''' - qdb.sql_connection.TRN.add(sql) - res = qdb.sql_connection.TRN.execute_fetchindex() + verbose: boolean, optional + Defaults to False. Prints status on what functin - for col_name in res: - col_name_list.append(col_name[1]) + """ + time = datetime.now().strftime('%m-%d-%y') - software_list = [s for s in qdb.software.Software.iter(active=active)] - software_commands = defaultdict(lambda: defaultdict( - lambda: defaultdict(list))) + # Retreave available col_name for commands + with qdb.sql_connection.TRN: + sql = 'SELECT col_name FROM qiita.resource_allocation_column_names;' + qdb.sql_connection.TRN.add(sql) + col_names_list = qdb.sql_connection.TRN.execute_fetchflatten() + # Retreave available software + software_list = list(qdb.software.Software.iter(active=active)) + scommands = {} for software in software_list: sname = software.name sversion = software.version - for command in software.commands: - cmd_name = command.name - for col in col_name_list: - software_commands[sname][sversion][cmd_name].append(col) - - final_obj = { - sname: { - sversion: dict(commands) - for sversion, commands in dict(versions).items() - } - for sname, versions in dict(software_commands).items() - } - return final_obj - - -def update_resource_allocation_redis(active=True): - """Updates redis with plots and information about current software. - - Parameters - ---------- - active: boolean, optional - Defaults to True. Should only be False when testing. + if sname not in scommands: + scommands[sname] = {} - """ - time = datetime.now().strftime('%m-%d-%y') + if sversion not in scommands[sname]: + scommands[sname][sversion] = {} - scommands = _build_software_commands_col_names_object(active) + for command in software.commands: + cmd_name = command.name + scommands[sname][sversion][cmd_name] = col_names_list redis_key = 'resources:commands' r_client.set(redis_key, str(scommands)) @@ -615,22 +599,23 @@ def update_resource_allocation_redis(active=True): for version, commands in versions.items(): for cname, col_name_list in commands.items(): df = retrieve_resource_data(cname, sname, version, COLUMNS) - print(("Retrieving allocation resources for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}")) - if len(df) == 0: - print(("No allocation resources available for" + + if verbose: + print(("Retrieving allocation resources for " + f" software: {sname}" + f" version: {version}" + f" command: {cname}")) + if len(df) == 0: + if verbose: + print(("No allocation resources available for" + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}")) continue # column_name_str looks like col1*col2*col3, etc for col_name_str in col_name_list: - df_copy = df.copy() new_column = None col_name_split = col_name_str.split('*') - df_copy.dropna(subset=col_name_split, inplace=True) + df_copy = df.dropna(subset=col_name_split) # Create a column with the desired columns for curr_column in col_name_split: @@ -638,15 +623,17 @@ def update_resource_allocation_redis(active=True): new_column = df_copy[curr_column] else: new_column *= df_copy[curr_column] - print(("Building resource allocation plot for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name_str}")) + if verbose: + print(("Building resource allocation plot for " + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) fig, axs = resource_allocation_plot(df_copy, col_name_str, - new_column) + new_column, + verbose=verbose) titles = [0, 0] images = [0, 0] @@ -697,11 +684,12 @@ def update_resource_allocation_redis(active=True): ("title_mem", titles[0], r_client.set), ("title_time", titles[1], r_client.set) ] - print(("Saving resource allocation image for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name_str}")) + if verbose: + print(("Saving resource allocation image for " + + f" software: {sname}" + + f" version: {version}" + + f" command: {cname}" + + f" column name: {col_name_str}")) for k, v, f in values: redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( diff --git a/qiita_db/util.py b/qiita_db/util.py index 1d7f2721b..1d819f540 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2326,7 +2326,7 @@ def send_email(to, subject, body): smtp.close() -def resource_allocation_plot(df, col_name_str, curr_column): +def resource_allocation_plot(df, col_name_str, curr_column, verbose=False): """Builds resource allocation plot for given filename and jobs Parameters @@ -2350,15 +2350,17 @@ def resource_allocation_plot(df, col_name_str, curr_column): df[col_name_str] = curr_column # models for memory - print("\tCalculating best model for memory") + if verbose: + print("\tCalculating best model for memory") _resource_allocation_plot_helper( - df, ax, "MaxRSSRaw", mem_models, col_name_str) + df, ax, "MaxRSSRaw", mem_models, col_name_str, verbose=verbose) ax = axs[1] # models for time - print("\tCalculating best model for time") + if verbose: + print("\tCalculating best model for time") _resource_allocation_plot_helper( - df, ax, "ElapsedRaw", time_models, col_name_str) + df, ax, "ElapsedRaw", time_models, col_name_str, verbose=verbose) return fig, axs @@ -2366,6 +2368,9 @@ def _retrieve_equations(): ''' Helper function for resource_allocation_plot. Retrieves equations from db. Creates dictionary for memory and time models. + This function is needed because it utilizes np as a part of eval() below. + In test_util.py we need to retrieve equations without importing np to + comply with PEP8 styling standard. Returns ------- @@ -2379,19 +2384,20 @@ def _retrieve_equations(): time_models = {} res = [] with qdb.sql_connection.TRN: - sql = ''' SELECT * FROM qiita.resource_allocation_equations; ''' + sql = '''SELECT equation_name, expression + FROM qiita.resource_allocation_equations;''' qdb.sql_connection.TRN.add(sql) res = qdb.sql_connection.TRN.execute_fetchindex() for models in res: - if 'mem' in models[1]: - memory_models[models[1]] = { - "equation_name": models[2], - "equation": lambda x, k, a, b: eval(models[2]) + if 'mem' in models[0]: + memory_models[models[0]] = { + "equation_name": models[1], + "equation": lambda x, k, a, b: eval(models[1]) } else: - time_models[models[1]] = { - "equation_name": models[2], - "equation": lambda x, k, a, b: eval(models[2]) + time_models[models[0]] = { + "equation_name": models[1], + "equation": lambda x, k, a, b: eval(models[1]) } return (memory_models, time_models) @@ -2452,7 +2458,7 @@ def retrieve_resource_data(cname, sname, version, columns): def _resource_allocation_plot_helper( - df, ax, curr, models, col_name): + df, ax, curr, models, col_name, verbose=False): """Helper function for resource allocation plot. Builds plot for MaxRSSRaw and ElapsedRaw @@ -2517,15 +2523,14 @@ def _resource_allocation_plot_helper( ax.set_ylabel(curr) ax.set_xlabel(col_name) - print(f"\t\tFitting best model for {curr}; column {col_name}") + if verbose: + print(f"\t\tFitting best model for {curr}; column {col_name}") # 50 - number of maximum iterations, 3 - number of failures we tolerate best_model_name, best_model, options = _resource_allocation_calculate( df, x_data, y_data, models, curr, col_name, 50, 3) - if options is None: - print(df) - print(x_data) - print(y_data) - print(f"\t\tSuccessfully chose best model for {curr}; column {col_name}") + if verbose: + print( + f"\t\tSuccessfully chose best model for {curr}; column {col_name}") k, a, b = options.x x_plot = np.array(sorted(df[col_name].unique())) y_plot = best_model(x_plot, k, a, b) From 43e146313126c86abe56a48fb66b4ed2d8ac70fd Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 12 Feb 2025 13:31:41 -0800 Subject: [PATCH 15/17] Update meta_util.py --- qiita_db/meta_util.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 5304e976e..fa9d7aea1 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -564,7 +564,7 @@ def update_resource_allocation_redis(active=True, verbose=False): Defaults to True. Should only be False when testing. verbose: boolean, optional - Defaults to False. Prints status on what functin + Defaults to False. Prints status on what function """ time = datetime.now().strftime('%m-%d-%y') @@ -573,7 +573,7 @@ def update_resource_allocation_redis(active=True, verbose=False): with qdb.sql_connection.TRN: sql = 'SELECT col_name FROM qiita.resource_allocation_column_names;' qdb.sql_connection.TRN.add(sql) - col_names_list = qdb.sql_connection.TRN.execute_fetchflatten() + col_names = qdb.sql_connection.TRN.execute_fetchflatten() # Retreave available software software_list = list(qdb.software.Software.iter(active=active)) @@ -590,14 +590,14 @@ def update_resource_allocation_redis(active=True, verbose=False): for command in software.commands: cmd_name = command.name - scommands[sname][sversion][cmd_name] = col_names_list + scommands[sname][sversion][cmd_name] = col_names redis_key = 'resources:commands' r_client.set(redis_key, str(scommands)) for sname, versions in scommands.items(): for version, commands in versions.items(): - for cname, col_name_list in commands.items(): + for cname, col_names in commands.items(): df = retrieve_resource_data(cname, sname, version, COLUMNS) if verbose: print(("Retrieving allocation resources for " + @@ -612,9 +612,9 @@ def update_resource_allocation_redis(active=True, verbose=False): f" command: {cname}")) continue # column_name_str looks like col1*col2*col3, etc - for col_name_str in col_name_list: + for col_name in col_names: new_column = None - col_name_split = col_name_str.split('*') + col_name_split = col_name.split('*') df_copy = df.dropna(subset=col_name_split) # Create a column with the desired columns @@ -628,10 +628,10 @@ def update_resource_allocation_redis(active=True, verbose=False): f" software: {sname}" + f" version: {version}" + f" command: {cname}" + - f" column name: {col_name_str}")) + f" column name: {col_name}")) fig, axs = resource_allocation_plot(df_copy, - col_name_str, + col_name, new_column, verbose=verbose) titles = [0, 0] @@ -689,10 +689,10 @@ def update_resource_allocation_redis(active=True, verbose=False): f" software: {sname}" + f" version: {version}" + f" command: {cname}" + - f" column name: {col_name_str}")) + f" column name: {col_name}")) for k, v, f in values: redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( - cname, sname, version, col_name_str, k) + cname, sname, version, col_name, k) r_client.delete(redis_key) f(redis_key, v) From 4032ed5c8d89eec6ceeee9f6d17b314c093fc1f6 Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 12 Feb 2025 14:22:36 -0800 Subject: [PATCH 16/17] Update util.py --- qiita_db/util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qiita_db/util.py b/qiita_db/util.py index 1d819f540..f21c78b60 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2556,7 +2556,8 @@ def _resource_allocation_plot_helper( ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, label="failures") - success_df.loc[:, 'node_name'] = success_df['node_name'].fillna('unknown') + success_df['node_name'].fillna('unknown', inplace=True) + slurm_hosts = set(success_df['node_name'].tolist()) cmap = colormaps['Accent'] if len(slurm_hosts) > len(cmap.colors): @@ -2763,8 +2764,8 @@ def _resource_allocation_success_failures(df, k, a, b, model, col_name, type_): x_plot = np.array(df[col_name]) df[f'c{type_}'] = model(x_plot, k, a, b) - success_df = df[df[type_] <= df[f'c{type_}']] - failures_df = df[df[type_] > df[f'c{type_}']] + success_df = df[df[type_] <= df[f'c{type_}']].copy() + failures_df = df[df[type_] > df[f'c{type_}']].copy() return (success_df, failures_df) From 8e148b6c94428d7143d9cad8d59a9f1ece9c957f Mon Sep 17 00:00:00 2001 From: gossty Date: Tue, 25 Feb 2025 11:03:12 -0800 Subject: [PATCH 17/17] Time limit for resource_allocation_plot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Created time limit for resource_allocation_plot using signal library. 2. Don’t show failures in legend if no failures. 3. Added more robust print statements with timestamps. 4. Only populate commands to qiita_pet when plots were created. --- qiita_db/meta_util.py | 93 +++++++++++++++++++++++++++++++------------ qiita_db/util.py | 20 ++++++---- 2 files changed, 81 insertions(+), 32 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index fa9d7aea1..c0df0fe89 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -36,6 +36,8 @@ from hashlib import md5 from re import sub from json import loads, dump, dumps +import signal +import traceback from qiita_db.util import create_nested_path, retrieve_resource_data from qiita_db.util import resource_allocation_plot @@ -555,7 +557,8 @@ def generate_plugin_releases(): f(redis_key, v) -def update_resource_allocation_redis(active=True, verbose=False): +def update_resource_allocation_redis(active=True, verbose=False, + time_limit=300): """Updates redis with plots and information about current software. Parameters @@ -564,7 +567,11 @@ def update_resource_allocation_redis(active=True, verbose=False): Defaults to True. Should only be False when testing. verbose: boolean, optional - Defaults to False. Prints status on what function + Defaults to False. Prints status on what function is running. + + time_limit: integer, optional + Defaults to 300, representing 5 minutes. This is the limit for how long + resource_allocation_plot function will run. """ time = datetime.now().strftime('%m-%d-%y') @@ -592,24 +599,24 @@ def update_resource_allocation_redis(active=True, verbose=False): cmd_name = command.name scommands[sname][sversion][cmd_name] = col_names - redis_key = 'resources:commands' - r_client.set(redis_key, str(scommands)) - + # software commands for which resource allocations were sucessfully + # calculated + scommands_allocation = {} for sname, versions in scommands.items(): for version, commands in versions.items(): for cname, col_names in commands.items(): df = retrieve_resource_data(cname, sname, version, COLUMNS) if verbose: - print(("Retrieving allocation resources for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}")) + print(("\nRetrieving allocation resources for:\n" + + f" software: {sname}\n" + + f" version: {version}\n" + + f" command: {cname}")) if len(df) == 0: if verbose: - print(("No allocation resources available for" + + print(("\nNo allocation resources available for" + f" software: {sname}" + f" version: {version}" + - f" command: {cname}")) + f" command: {cname}\n")) continue # column_name_str looks like col1*col2*col3, etc for col_name in col_names: @@ -624,16 +631,38 @@ def update_resource_allocation_redis(active=True, verbose=False): else: new_column *= df_copy[curr_column] if verbose: - print(("Building resource allocation plot for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name}")) + print( + ("\nBuilding resource allocation plot for:\n" + + f" software: {sname}\n" + + f" version: {version}\n" + + f" command: {cname}\n" + + f" column name: {col_name}\n" + + f" {datetime.now().strftime('%b %d %H:%M:%S')}")) + + def timeout_handler(signum, frame): + raise TimeoutError(( + "\nresource_allocation_plot " + + "execution exceeded time limit." + + "For:\n" + f" software: {sname}\n" + + f" version: {version}\n" + + f" command: {cname}\n" + + f" column name: {col_name}\n" + + f" {datetime.now().strftime('%b %d %H:%M:%S')}")) + + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(time_limit) + try: + fig, axs = resource_allocation_plot(df_copy, + col_name, + new_column, + verbose=verbose) + signal.alarm(0) + except TimeoutError: + print("Timeout reached!") + traceback.print_exc() + continue - fig, axs = resource_allocation_plot(df_copy, - col_name, - new_column, - verbose=verbose) titles = [0, 0] images = [0, 0] @@ -685,14 +714,28 @@ def update_resource_allocation_redis(active=True, verbose=False): ("title_time", titles[1], r_client.set) ] if verbose: - print(("Saving resource allocation image for " + - f" software: {sname}" + - f" version: {version}" + - f" command: {cname}" + - f" column name: {col_name}")) + print( + ("Saving resource allocation image for\n" + + f" software: {sname}\n" + + f" version: {version}\n" + + f" command: {cname}\n" + + f" column name: {col_name}\n" + + f" {datetime.now().strftime('%b %d %H:%M:%S')}")) for k, v, f in values: redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( cname, sname, version, col_name, k) r_client.delete(redis_key) f(redis_key, v) + + if sname not in scommands_allocation: + scommands_allocation[sname] = {} + if version not in scommands_allocation[sname]: + scommands_allocation[sname][version] = {} + if cname not in scommands_allocation[sname][version]: + scommands_allocation[sname][version][cname] = [] + scommands_allocation[sname][version][cname].append( + col_name) + + redis_key = 'resources:commands' + r_client.set(redis_key, str(scommands_allocation)) diff --git a/qiita_db/util.py b/qiita_db/util.py index f21c78b60..616388277 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2527,7 +2527,7 @@ def _resource_allocation_plot_helper( print(f"\t\tFitting best model for {curr}; column {col_name}") # 50 - number of maximum iterations, 3 - number of failures we tolerate best_model_name, best_model, options = _resource_allocation_calculate( - df, x_data, y_data, models, curr, col_name, 50, 3) + df, x_data, y_data, models, curr, col_name, 50, 3, verbose) if verbose: print( f"\t\tSuccessfully chose best model for {curr}; column {col_name}") @@ -2553,8 +2553,9 @@ def _resource_allocation_plot_helper( success_df, failures_df = _resource_allocation_success_failures( df, k, a, b, best_model, col_name, curr) failures = failures_df.shape[0] - ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, - label="failures") + if failures != 0: + ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, + label="failures") success_df['node_name'].fillna('unknown', inplace=True) @@ -2580,7 +2581,7 @@ def _resource_allocation_plot_helper( def _resource_allocation_calculate( - df, x, y, models, type_, col_name, depth, tolerance): + df, x, y, models, type_, col_name, depth, tolerance, verbose): """Helper function for resource allocation plot. Calculates best_model and best_result given the models list and x,y data. @@ -2621,6 +2622,11 @@ def _resource_allocation_calculate( best_failures = np.inf best_max = np.inf for model_name, model in models.items(): + if verbose: + print( + f"\t\t\tCalculating {model_name} for {type_}; " + f"{col_name} {datetime.now().strftime('%b %d %H:%M:%S')}" + ) model_equation = model['equation'] # start values for binary search, where sl is left, sr is right # penalty weight must be positive & non-zero, hence, sl >= 1. @@ -2763,9 +2769,9 @@ def _resource_allocation_success_failures(df, k, a, b, model, col_name, type_): """ x_plot = np.array(df[col_name]) - df[f'c{type_}'] = model(x_plot, k, a, b) - success_df = df[df[type_] <= df[f'c{type_}']].copy() - failures_df = df[df[type_] > df[f'c{type_}']].copy() + y_plot = model(x_plot, k, a, b) + success_df = df[df[type_] <= y_plot].copy() + failures_df = df[df[type_] > y_plot].copy() return (success_df, failures_df)